In [1]:
import sys
import platform
if platform.system() == 'Linux':
    sys.path.append('/home/huiying/Dropbox/Program/')
elif platform.system() == 'Windows':
    sys.path.append('D:/Dropbox/Program/')
        
from own_packages.SN import *
from own_packages.preprocess import *
from own_packages.Feature_Selection import *
from own_packages.Final_Model import *

import os
from numpy import *
from pandas import *
import matplotlib
import matplotlib.pyplot as plt

from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier as RF

%matplotlib inline
In [25]:
files = os.listdir()

# read the label(y)
y_all = read_csv('y_all.csv',header=0,index_col=0)
y_all = y_all.label
print(y_all)

# generate or read the No. list of cross-validation
if 'OneCV_ID_Test_CV.csv' in files and 'OneCV_ID_Train_CV.csv' in files:
    ID_train = read_csv('OneCV_ID_Train_CV.csv',header=0,index_col=0)
    ID_test = read_csv('OneCV_ID_Test_CV.csv',header=0,index_col=0)
else:
    ID_train,ID_test = get_CVID_CSV(y_all,n_splits=5,name='OneCV')
    
# set the default parameters of plot
plot_default()

# set the algorithms
lr = LogisticRegression(solver='liblinear')
svc = svm.SVC(probability=True,gamma='scale')
rf = RF(n_estimators=100)
MLID
1234026    0
1290264    0
1299628    0
1313585    1
1319491    0
1319857    1
1340213    0
1345116    1
1347742    0
1360543    0
1361111    0
1363165    0
1404820    1
1412799    0
1422362    1
1474840    0
1475214    0
1476232    0
1482287    0
1487216    0
1494519    0
1503424    0
1505342    0
1506433    0
1513571    0
1516688    1
1520407    1
1530123    0
1571633    0
1584572    0
          ..
2588525    1
2592990    0
2594215    1
2610058    0
2628972    0
2642645    0
2685459    0
2693184    0
2712262    1
2722506    1
2809900    1
2825428    0
2832556    1
2845748    1
2847171    1
2872625    0
2885862    0
2916823    0
2935369    0
2944076    1
2945181    0
2947334    1
3014537    1
3029304    1
3062868    1
3067890    0
3093550    1
3100166    1
3105376    0
3113885    1
Name: label, Length: 124, dtype: int64
In [28]:
# set the name of dirs and names
fpath_out = 'T2'
name = 'T2'

# read and standardize the X 
if 'T2' in files:
    X_T2_Sd = read_csv('T2/X_T2_Sd.csv',header=0,index_col=0)
else:
    if not os.path.exists(fpath_out):os.makedirs(fpath_out)
    X_T2 = read_csv('X_all_T2.csv',header=0,index_col=0)
    # X_T2_Sd = Std_features(X_T2)
    # X_T2_Sd.to_csv('T2/X_T2_Sd.csv')
    
# set the structure of inputs
fpath_out = 'T2/LASSO'
if not os.path.exists(fpath_out):os.makedirs(fpath_out)
T2_input = paras(X=X_T2,y=y_all,ID_test=ID_test,ID_train=ID_train,
                 name=name,fpath=fpath_out)

# the first step of feature selection 
T2_output_Lasso = LASSO_CV.selection(T2_input,inner_n_splits=10)
plt.close('all')

# model developing
T2_output_Further = T2_output_Lasso
for clf,model_name in [(lr,'LR'),(svc,'SVM'),(rf,'RF')]:
    T2_output_Lasso.model = clf
    T2_output_Lasso.name = model_name
    T2_output_Lasso.fpath = fpath_out
    T2_results_CV,T2_result_all = ModelPred.OneCV(T2_output_Lasso,label=['Neg','Pos'])
    plt.close()
-----Start LASSO_CV-----
Alpha: 0.07374151122277184
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
Features reduced from 1029 to 6 by LASSO
                                 Coefficients
Features                                     
wavelet-HHL_firstorder_Median       -0.080493
wavelet-LHL_firstorder_Kurtosis     -0.008507
wavelet-LLL_glcm_Imc1                0.001751
wavelet-LHL_firstorder_Skewness      0.007827
original_shape_MinorAxis             0.013365
original_shape_SurfaceArea           0.134488
-----0-----
Alpha: 0.07999008066126778

Features reduced from 1029 to 5 by LASSO
                                 Coefficients
Features                                     
wavelet-HHL_firstorder_Median       -0.051056
wavelet-LHL_firstorder_Kurtosis     -0.009830
original_shape_Compactness2         -0.005178
original_shape_MinorAxis             0.013909
original_shape_SurfaceArea           0.154586
-----1-----
Alpha: 0.0823511334487696
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
Features reduced from 1029 to 5 by LASSO
                                                    Coefficients
Features                                                        
wavelet-HHL_firstorder_Median                          -0.058846
exponential_glszm_GrayLevelNonUniformityNormalized      0.006572
wavelet-HHL_glszm_LargeAreaHighGrayLevelEmphasis        0.009509
original_shape_MinorAxis                                0.018076
original_shape_SurfaceArea                              0.138468
-----2-----
Alpha: 0.10937061133294382
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
Features reduced from 1029 to 3 by LASSO
                               Coefficients
Features                                   
wavelet-HHL_firstorder_Median     -0.004822
original_shape_MinorAxis           0.026952
original_shape_SurfaceArea         0.085461
-----3-----
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
Alpha: 0.06076732428711335
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
Features reduced from 1029 to 9 by LASSO
                                                  Coefficients
Features                                                      
wavelet-HHL_firstorder_Median                        -0.071032
original_shape_Compactness2                          -0.013200
wavelet-HLH_firstorder_Skewness                      -0.004270
wavelet-HLL_firstorder_Skewness                       0.004617
wavelet-HHL_glszm_LargeAreaHighGrayLevelEmphasis      0.010977
original_glszm_SmallAreaLowGrayLevelEmphasis          0.012407
wavelet-HHL_glrlm_RunVariance                         0.025580
wavelet-LHL_firstorder_Skewness                       0.033874
original_shape_SurfaceArea                            0.165685
-----4-----
-----Finish!-----
-----Start Model-----
-----0-----
------test------
              precision    recall  f1-score   support

         Neg       0.88      1.00      0.93        14
         Pos       1.00      0.82      0.90        11

   micro avg       0.92      0.92      0.92        25
   macro avg       0.94      0.91      0.92        25
weighted avg       0.93      0.92      0.92        25

[Confusion Matrix]
TN	FP
FN	TP
[[14  0]
 [ 2  9]] 

cutoff = ('Youden', 0.4267072366789403)
recall = 0.8181818181818182
precision = 1.0
sensitivity = 0.8181818181818182
specificity = 1.0
accuracy = 0.92
F1 = 0.9
brier = 0.14225119104950892
AUC = 0.922077922077922
95%CI-AUC = ('Binomial', 0.7425027172678562, 0.9908159374803416)

------train------
              precision    recall  f1-score   support

         Neg       0.88      0.82      0.85        56
         Pos       0.79      0.86      0.82        43

   micro avg       0.84      0.84      0.84        99
   macro avg       0.84      0.84      0.84        99
weighted avg       0.84      0.84      0.84        99

[Confusion Matrix]
TN	FP
FN	TP
[[46 10]
 [ 6 37]] 

cutoff = ('Youden', 0.5219271690291363)
recall = 0.8604651162790697
precision = 0.7872340425531915
sensitivity = 0.8604651162790697
specificity = 0.8214285714285714
accuracy = 0.8383838383838383
F1 = 0.8222222222222222
brier = 0.13948373768088151
AUC = 0.880813953488372
95%CI-AUC = ('Binomial', 0.8002426948579309, 0.937276419220511)

-----1-----
------test------
              precision    recall  f1-score   support

         Neg       1.00      0.57      0.73        14
         Pos       0.65      1.00      0.79        11

   micro avg       0.76      0.76      0.76        25
   macro avg       0.82      0.79      0.76        25
weighted avg       0.84      0.76      0.75        25

[Confusion Matrix]
TN	FP
FN	TP
[[ 8  6]
 [ 0 11]] 

cutoff = ('Youden', 0.42278606692802445)
recall = 1.0
precision = 0.6470588235294118
sensitivity = 1.0
specificity = 0.5714285714285714
accuracy = 0.76
F1 = 0.7857142857142858
brier = 0.21611177912884738
AUC = 0.7207792207792207
95%CI-AUC = ('Binomial', 0.506938318533891, 0.8798285008906752)

------train------
              precision    recall  f1-score   support

         Neg       0.92      0.84      0.88        56
         Pos       0.81      0.91      0.86        43

   micro avg       0.87      0.87      0.87        99
   macro avg       0.87      0.87      0.87        99
weighted avg       0.87      0.87      0.87        99

[Confusion Matrix]
TN	FP
FN	TP
[[47  9]
 [ 4 39]] 

cutoff = ('Youden', 0.434834054339196)
recall = 0.9069767441860465
precision = 0.8125
sensitivity = 0.9069767441860465
specificity = 0.8392857142857143
accuracy = 0.8686868686868687
F1 = 0.8571428571428572
brier = 0.12520390307881762
AUC = 0.9003322259136213
95%CI-AUC = ('Binomial', 0.8237075049706944, 0.9514430078931789)

-----2-----
------test------
              precision    recall  f1-score   support

         Neg       0.82      0.64      0.72        14
         Pos       0.64      0.82      0.72        11

   micro avg       0.72      0.72      0.72        25
   macro avg       0.73      0.73      0.72        25
weighted avg       0.74      0.72      0.72        25

[Confusion Matrix]
TN	FP
FN	TP
[[9 5]
 [2 9]] 

cutoff = ('Youden', 0.4408220182667646)
recall = 0.8181818181818182
precision = 0.6428571428571429
sensitivity = 0.8181818181818182
specificity = 0.6428571428571429
accuracy = 0.72
F1 = 0.7200000000000001
brier = 0.21619411310923273
AUC = 0.7142857142857143
95%CI-AUC = ('Binomial', 0.5001623351600223, 0.8752665811796557)

------train------
              precision    recall  f1-score   support

         Neg       0.92      0.80      0.86        56
         Pos       0.78      0.91      0.84        43

   micro avg       0.85      0.85      0.85        99
   macro avg       0.85      0.86      0.85        99
weighted avg       0.86      0.85      0.85        99

[Confusion Matrix]
TN	FP
FN	TP
[[45 11]
 [ 4 39]] 

cutoff = ('Youden', 0.4224025188527173)
recall = 0.9069767441860465
precision = 0.78
sensitivity = 0.9069767441860465
specificity = 0.8035714285714286
accuracy = 0.8484848484848485
F1 = 0.8387096774193548
brier = 0.12760580917330744
AUC = 0.8982558139534883
95%CI-AUC = ('Binomial', 0.8211841253135906, 0.9499661051802076)

-----3-----
------test------
              precision    recall  f1-score   support

         Neg       1.00      0.79      0.88        14
         Pos       0.79      1.00      0.88        11

   micro avg       0.88      0.88      0.88        25
   macro avg       0.89      0.89      0.88        25
weighted avg       0.91      0.88      0.88        25

[Confusion Matrix]
TN	FP
FN	TP
[[11  3]
 [ 0 11]] 

cutoff = ('Youden', 0.33555088208194583)
recall = 1.0
precision = 0.7857142857142857
sensitivity = 1.0
specificity = 0.7857142857142857
accuracy = 0.88
F1 = 0.88
brier = 0.12476560348807517
AUC = 0.9545454545454546
95%CI-AUC = ('Binomial', 0.7883419356554456, 0.9983179285128623)

------train------
              precision    recall  f1-score   support

         Neg       0.82      0.84      0.83        56
         Pos       0.79      0.77      0.78        43

   micro avg       0.81      0.81      0.81        99
   macro avg       0.81      0.80      0.80        99
weighted avg       0.81      0.81      0.81        99

[Confusion Matrix]
TN	FP
FN	TP
[[47  9]
 [10 33]] 

cutoff = ('Youden', 0.491060078753171)
recall = 0.7674418604651163
precision = 0.7857142857142857
sensitivity = 0.7674418604651163
specificity = 0.8392857142857143
accuracy = 0.8080808080808081
F1 = 0.7764705882352941
brier = 0.16721251866738704
AUC = 0.8251661129568106
95%CI-AUC = ('Binomial', 0.7358328768495677, 0.8941670790806746)

-----4-----
------test------
              precision    recall  f1-score   support

         Neg       0.82      0.64      0.72        14
         Pos       0.62      0.80      0.70        10

   micro avg       0.71      0.71      0.71        24
   macro avg       0.72      0.72      0.71        24
weighted avg       0.73      0.71      0.71        24

[Confusion Matrix]
TN	FP
FN	TP
[[9 5]
 [2 8]] 

cutoff = ('Youden', 0.43066135782392717)
recall = 0.8
precision = 0.6153846153846154
sensitivity = 0.8
specificity = 0.6428571428571429
accuracy = 0.7083333333333334
F1 = 0.6956521739130435
brier = 0.24367684314974705
AUC = 0.7285714285714285
95%CI-AUC = ('Binomial', 0.510137636928788, 0.8878833127936933)

------train------
              precision    recall  f1-score   support

         Neg       0.94      0.79      0.85        56
         Pos       0.77      0.93      0.85        44

   micro avg       0.85      0.85      0.85       100
   macro avg       0.85      0.86      0.85       100
weighted avg       0.86      0.85      0.85       100

[Confusion Matrix]
TN	FP
FN	TP
[[44 12]
 [ 3 41]] 

cutoff = ('Youden', 0.39405096960460056)
recall = 0.9318181818181818
precision = 0.7735849056603774
sensitivity = 0.9318181818181818
specificity = 0.7857142857142857
accuracy = 0.85
F1 = 0.8453608247422681
brier = 0.12052680266926385
AUC = 0.9119318181818182
95%CI-AUC = ('Binomial', 0.8384028015162921, 0.9593496496395022)

                    0         1         2         3         4      Mean  \
recall       0.860465  0.906977  0.906977  0.767442  0.931818  0.874736   
precision    0.787234  0.812500  0.780000  0.785714  0.773585  0.787807   
sensitivity  0.860465  0.906977  0.906977  0.767442  0.931818  0.874736   
specificity  0.821429  0.839286  0.803571  0.839286  0.785714  0.817857   
accuracy     0.838384  0.868687  0.848485  0.808081  0.850000  0.842727   
F1           0.822222  0.857143  0.838710  0.776471  0.845361  0.827981   
brier        0.139484  0.125204  0.127606  0.167213  0.120527  0.136007   
AUC          0.880814  0.900332  0.898256  0.825166  0.911932  0.883300   

                  Std  
recall       0.065294  
precision    0.014813  
sensitivity  0.065294  
specificity  0.023283  
accuracy     0.022244  
F1           0.031441  
brier        0.018794  
AUC          0.034349  
                    0         1         2         3         4      Mean  \
recall       0.818182  1.000000  0.818182  1.000000  0.800000  0.887273   
precision    1.000000  0.647059  0.642857  0.785714  0.615385  0.738203   
sensitivity  0.818182  1.000000  0.818182  1.000000  0.800000  0.887273   
specificity  1.000000  0.571429  0.642857  0.785714  0.642857  0.728571   
accuracy     0.920000  0.760000  0.720000  0.880000  0.708333  0.797667   
F1           0.900000  0.785714  0.720000  0.880000  0.695652  0.796273   
brier        0.142251  0.216112  0.216194  0.124766  0.243677  0.188600   
AUC          0.922078  0.720779  0.714286  0.954545  0.728571  0.808052   

                  Std  
recall       0.103173  
precision    0.160684  
sensitivity  0.103173  
specificity  0.170533  
accuracy     0.096405  
F1           0.091955  
brier        0.051901  
AUC          0.119570  
------all------
              precision    recall  f1-score   support

         Neg       0.87      0.74      0.80        70
         Pos       0.72      0.85      0.78        54

   micro avg       0.79      0.79      0.79       124
   macro avg       0.79      0.80      0.79       124
weighted avg       0.80      0.79      0.79       124

[Confusion Matrix]
TN	FP
FN	TP
[[52 18]
 [ 8 46]] 

cutoff = ('Youden', 0.41835055833140555)
recall = 0.8518518518518519
precision = 0.71875
sensitivity = 0.8518518518518519
specificity = 0.7428571428571429
accuracy = 0.7903225806451613
F1 = 0.7796610169491525
brier = 0.1881557371369801
AUC = 0.7891534391534393
95%CI-AUC = ('Binomial', 0.7067808101740262, 0.8572289148665777)

-----Finish!-----
-----Start Model-----
-----0-----
------test------
              precision    recall  f1-score   support

         Neg       0.86      0.86      0.86        14
         Pos       0.82      0.82      0.82        11

   micro avg       0.84      0.84      0.84        25
   macro avg       0.84      0.84      0.84        25
weighted avg       0.84      0.84      0.84        25

[Confusion Matrix]
TN	FP
FN	TP
[[12  2]
 [ 2  9]] 

cutoff = ('Youden', 0.42616384289488374)
recall = 0.8181818181818182
precision = 0.8181818181818182
sensitivity = 0.8181818181818182
specificity = 0.8571428571428571
accuracy = 0.84
F1 = 0.8181818181818182
brier = 0.17626465964922364
AUC = 0.7922077922077921
95%CI-AUC = ('Binomial', 0.5841998281075556, 0.9269351737428029)

------train------
              precision    recall  f1-score   support

         Neg       0.96      0.79      0.86        56
         Pos       0.77      0.95      0.85        43

   micro avg       0.86      0.86      0.86        99
   macro avg       0.87      0.87      0.86        99
weighted avg       0.88      0.86      0.86        99

[Confusion Matrix]
TN	FP
FN	TP
[[44 12]
 [ 2 41]] 

cutoff = ('Youden', 0.36059121670305017)
recall = 0.9534883720930233
precision = 0.7735849056603774
sensitivity = 0.9534883720930233
specificity = 0.7857142857142857
accuracy = 0.8585858585858586
F1 = 0.8541666666666667
brier = 0.1126683474014466
AUC = 0.9102990033222591
95%CI-AUC = ('Binomial', 0.8359207439334404, 0.958418504744027)

-----1-----
------test------
              precision    recall  f1-score   support

         Neg       0.89      0.57      0.70        14
         Pos       0.62      0.91      0.74        11

   micro avg       0.72      0.72      0.72        25
   macro avg       0.76      0.74      0.72        25
weighted avg       0.77      0.72      0.72        25

[Confusion Matrix]
TN	FP
FN	TP
[[ 8  6]
 [ 1 10]] 

cutoff = ('Youden', 0.5)
recall = 0.9090909090909091
precision = 0.625
sensitivity = 0.9090909090909091
specificity = 0.5714285714285714
accuracy = 0.72
F1 = 0.7407407407407406
brier = 0.20927435962451896
AUC = 0.7012987012987013
95%CI-AUC = ('Binomial', 0.4867234203121368, 0.8660179541627924)

------train------
              precision    recall  f1-score   support

         Neg       0.84      0.96      0.90        56
         Pos       0.94      0.77      0.85        43

   micro avg       0.88      0.88      0.88        99
   macro avg       0.89      0.87      0.87        99
weighted avg       0.89      0.88      0.88        99

[Confusion Matrix]
TN	FP
FN	TP
[[54  2]
 [10 33]] 

cutoff = ('Youden', 0.615372892842085)
recall = 0.7674418604651163
precision = 0.9428571428571428
sensitivity = 0.7674418604651163
specificity = 0.9642857142857143
accuracy = 0.8787878787878788
F1 = 0.8461538461538461
brier = 0.10293226916354008
AUC = 0.9227574750830565
95%CI-AUC = ('Binomial', 0.8514499690678909, 0.9668390113927129)

-----2-----
------test------
              precision    recall  f1-score   support

         Neg       0.85      0.79      0.81        14
         Pos       0.75      0.82      0.78        11

   micro avg       0.80      0.80      0.80        25
   macro avg       0.80      0.80      0.80        25
weighted avg       0.80      0.80      0.80        25

[Confusion Matrix]
TN	FP
FN	TP
[[11  3]
 [ 2  9]] 

cutoff = ('Youden', 0.5184348239523857)
recall = 0.8181818181818182
precision = 0.75
sensitivity = 0.8181818181818182
specificity = 0.7857142857142857
accuracy = 0.8
F1 = 0.7826086956521738
brier = 0.17410291442452358
AUC = 0.8116883116883117
95%CI-AUC = ('Binomial', 0.6062459490380402, 0.9386536573797263)

------train------
              precision    recall  f1-score   support

         Neg       0.89      0.88      0.88        56
         Pos       0.84      0.86      0.85        43

   micro avg       0.87      0.87      0.87        99
   macro avg       0.87      0.87      0.87        99
weighted avg       0.87      0.87      0.87        99

[Confusion Matrix]
TN	FP
FN	TP
[[49  7]
 [ 6 37]] 

cutoff = ('Youden', 0.48787607002262756)
recall = 0.8604651162790697
precision = 0.8409090909090909
sensitivity = 0.8604651162790697
specificity = 0.875
accuracy = 0.8686868686868687
F1 = 0.8505747126436781
brier = 0.12070078306976685
AUC = 0.9028239202657807
95%CI-AUC = ('Binomial', 0.8267448381442899, 0.9532048874802718)

-----3-----
------test------
              precision    recall  f1-score   support

         Neg       0.92      0.79      0.85        14
         Pos       0.77      0.91      0.83        11

   micro avg       0.84      0.84      0.84        25
   macro avg       0.84      0.85      0.84        25
weighted avg       0.85      0.84      0.84        25

[Confusion Matrix]
TN	FP
FN	TP
[[11  3]
 [ 1 10]] 

cutoff = ('Youden', 0.2534909236873359)
recall = 0.9090909090909091
precision = 0.7692307692307693
sensitivity = 0.9090909090909091
specificity = 0.7857142857142857
accuracy = 0.84
F1 = 0.8333333333333333
brier = 0.1350683289008123
AUC = 0.9285714285714285
95%CI-AUC = ('Binomial', 0.7513653244637356, 0.9927435353916194)

------train------
              precision    recall  f1-score   support

         Neg       0.80      0.88      0.84        56
         Pos       0.82      0.72      0.77        43

   micro avg       0.81      0.81      0.81        99
   macro avg       0.81      0.80      0.80        99
weighted avg       0.81      0.81      0.81        99

[Confusion Matrix]
TN	FP
FN	TP
[[49  7]
 [12 31]] 

cutoff = ('Youden', 0.5242508293427226)
recall = 0.7209302325581395
precision = 0.8157894736842105
sensitivity = 0.7209302325581395
specificity = 0.875
accuracy = 0.8080808080808081
F1 = 0.765432098765432
brier = 0.153285283164566
AUC = 0.8666943521594684
95%CI-AUC = ('Binomial', 0.7835864224034667, 0.926677498055638)

-----4-----
------test------
              precision    recall  f1-score   support

         Neg       0.85      0.79      0.81        14
         Pos       0.73      0.80      0.76        10

   micro avg       0.79      0.79      0.79        24
   macro avg       0.79      0.79      0.79        24
weighted avg       0.80      0.79      0.79        24

[Confusion Matrix]
TN	FP
FN	TP
[[11  3]
 [ 2  8]] 

cutoff = ('Youden', 0.5)
recall = 0.8
precision = 0.7272727272727273
sensitivity = 0.8
specificity = 0.7857142857142857
accuracy = 0.7916666666666666
F1 = 0.761904761904762
brier = 0.18387271923771134
AUC = 0.7857142857142857
95%CI-AUC = ('Binomial', 0.5718546926357083, 0.9250456179687029)

------train------
              precision    recall  f1-score   support

         Neg       0.93      0.91      0.92        56
         Pos       0.89      0.91      0.90        44

   micro avg       0.91      0.91      0.91       100
   macro avg       0.91      0.91      0.91       100
weighted avg       0.91      0.91      0.91       100

[Confusion Matrix]
TN	FP
FN	TP
[[51  5]
 [ 4 40]] 

cutoff = ('Youden', 0.4836372877167067)
recall = 0.9090909090909091
precision = 0.8888888888888888
sensitivity = 0.9090909090909091
specificity = 0.9107142857142857
accuracy = 0.91
F1 = 0.8988764044943819
brier = 0.09107219405859732
AUC = 0.9594155844155844
95%CI-AUC = ('Binomial', 0.8999369465778991, 0.988694092758204)

                    0         1         2         3         4      Mean  \
recall       0.953488  0.767442  0.860465  0.720930  0.909091  0.842283   
precision    0.773585  0.942857  0.840909  0.815789  0.888889  0.852406   
sensitivity  0.953488  0.767442  0.860465  0.720930  0.909091  0.842283   
specificity  0.785714  0.964286  0.875000  0.875000  0.910714  0.882143   
accuracy     0.858586  0.878788  0.868687  0.808081  0.910000  0.864828   
F1           0.854167  0.846154  0.850575  0.765432  0.898876  0.843041   
brier        0.112668  0.102932  0.120701  0.153285  0.091072  0.116132   
AUC          0.910299  0.922757  0.902824  0.866694  0.959416  0.912398   

                  Std  
recall       0.096809  
precision    0.065571  
sensitivity  0.096809  
specificity  0.065124  
accuracy     0.037113  
F1           0.048298  
brier        0.023534  
AUC          0.033552  
                    0         1         2         3         4      Mean  \
recall       0.818182  0.909091  0.818182  0.909091  0.800000  0.850909   
precision    0.818182  0.625000  0.750000  0.769231  0.727273  0.737937   
sensitivity  0.818182  0.909091  0.818182  0.909091  0.800000  0.850909   
specificity  0.857143  0.571429  0.785714  0.785714  0.785714  0.757143   
accuracy     0.840000  0.720000  0.800000  0.840000  0.791667  0.798333   
F1           0.818182  0.740741  0.782609  0.833333  0.761905  0.787354   
brier        0.176265  0.209274  0.174103  0.135068  0.183873  0.175717   
AUC          0.792208  0.701299  0.811688  0.928571  0.785714  0.803896   

                  Std  
recall       0.053629  
precision    0.071471  
sensitivity  0.053629  
specificity  0.108327  
accuracy     0.049131  
F1           0.038430  
brier        0.026683  
AUC          0.081545  
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------all------
              precision    recall  f1-score   support

         Neg       0.85      0.79      0.81        70
         Pos       0.75      0.81      0.78        54

   micro avg       0.80      0.80      0.80       124
   macro avg       0.80      0.80      0.80       124
weighted avg       0.80      0.80      0.80       124

[Confusion Matrix]
TN	FP
FN	TP
[[55 15]
 [10 44]] 

cutoff = ('Youden', 0.42616384289488374)
recall = 0.8148148148148148
precision = 0.7457627118644068
sensitivity = 0.8148148148148148
specificity = 0.7857142857142857
accuracy = 0.7983870967741935
F1 = 0.7787610619469028
brier = 0.17565082118291966
AUC = 0.8034391534391534
95%CI-AUC = ('Binomial', 0.7225090298864401, 0.8693406224325474)

-----Finish!-----
-----Start Model-----
-----0-----
------test------
              precision    recall  f1-score   support

         Neg       0.75      0.86      0.80        14
         Pos       0.78      0.64      0.70        11

   micro avg       0.76      0.76      0.76        25
   macro avg       0.76      0.75      0.75        25
weighted avg       0.76      0.76      0.76        25

[Confusion Matrix]
TN	FP
FN	TP
[[12  2]
 [ 4  7]] 

cutoff = ('Youden', 0.55)
recall = 0.6363636363636364
precision = 0.7777777777777778
sensitivity = 0.6363636363636364
specificity = 0.8571428571428571
accuracy = 0.76
F1 = 0.7000000000000001
brier = 0.20026
AUC = 0.737012987012987
95%CI-AUC = ('Binomial', 0.524048524156825, 0.8910439384661436)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

         Neg       1.00      1.00      1.00        56
         Pos       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.7)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.02496363636363637
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

-----1-----
------test------
              precision    recall  f1-score   support

         Neg       0.83      0.71      0.77        14
         Pos       0.69      0.82      0.75        11

   micro avg       0.76      0.76      0.76        25
   macro avg       0.76      0.77      0.76        25
weighted avg       0.77      0.76      0.76        25

[Confusion Matrix]
TN	FP
FN	TP
[[10  4]
 [ 2  9]] 

cutoff = ('Youden', 0.61)
recall = 0.8181818181818182
precision = 0.6923076923076923
sensitivity = 0.8181818181818182
specificity = 0.7142857142857143
accuracy = 0.76
F1 = 0.7500000000000001
brier = 0.17698400000000003
AUC = 0.827922077922078
95%CI-AUC = ('Binomial', 0.6249889329318279, 0.9479750440455293)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

         Neg       1.00      1.00      1.00        56
         Pos       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.62)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.026535353535353537
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

-----2-----
------test------
              precision    recall  f1-score   support

         Neg       0.82      0.64      0.72        14
         Pos       0.64      0.82      0.72        11

   micro avg       0.72      0.72      0.72        25
   macro avg       0.73      0.73      0.72        25
weighted avg       0.74      0.72      0.72        25

[Confusion Matrix]
TN	FP
FN	TP
[[9 5]
 [2 9]] 

cutoff = ('Youden', 0.36)
recall = 0.8181818181818182
precision = 0.6428571428571429
sensitivity = 0.8181818181818182
specificity = 0.6428571428571429
accuracy = 0.72
F1 = 0.7200000000000001
brier = 0.207364
AUC = 0.724025974025974
95%CI-AUC = ('Binomial', 0.5103407260113572, 0.8820934725837822)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

         Neg       1.00      1.00      1.00        56
         Pos       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.62)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.02494141414141414
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

-----3-----
------test------
              precision    recall  f1-score   support

         Neg       0.92      0.79      0.85        14
         Pos       0.77      0.91      0.83        11

   micro avg       0.84      0.84      0.84        25
   macro avg       0.84      0.85      0.84        25
weighted avg       0.85      0.84      0.84        25

[Confusion Matrix]
TN	FP
FN	TP
[[11  3]
 [ 1 10]] 

cutoff = ('Youden', 0.29)
recall = 0.9090909090909091
precision = 0.7692307692307693
sensitivity = 0.9090909090909091
specificity = 0.7857142857142857
accuracy = 0.84
F1 = 0.8333333333333333
brier = 0.13916
AUC = 0.8733766233766235
95%CI-AUC = ('Binomial', 0.6795599341721734, 0.9714887603791876)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

         Neg       1.00      1.00      1.00        56
         Pos       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.58)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.028853535353535357
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

-----4-----
------test------
              precision    recall  f1-score   support

         Neg       0.79      0.79      0.79        14
         Pos       0.70      0.70      0.70        10

   micro avg       0.75      0.75      0.75        24
   macro avg       0.74      0.74      0.74        24
weighted avg       0.75      0.75      0.75        24

[Confusion Matrix]
TN	FP
FN	TP
[[11  3]
 [ 3  7]] 

cutoff = ('Youden', 0.62)
recall = 0.7
precision = 0.7
sensitivity = 0.7
specificity = 0.7857142857142857
accuracy = 0.75
F1 = 0.7
brier = 0.211275
AUC = 0.7285714285714285
95%CI-AUC = ('Binomial', 0.510137636928788, 0.8878833127936933)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

         Neg       1.00      1.00      1.00        56
         Pos       1.00      1.00      1.00        44

   micro avg       1.00      1.00      1.00       100
   macro avg       1.00      1.00      1.00       100
weighted avg       1.00      1.00      1.00       100

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 44]] 

cutoff = ('Youden', 0.66)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.024641000000000003
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9637833073548235, nan)

                    0         1         2         3         4      Mean  \
recall       1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
precision    1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
sensitivity  1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
specificity  1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
accuracy     1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
F1           1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
brier        0.024964  0.026535  0.024941  0.028854  0.024641  0.025987   
AUC          1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   

                  Std  
recall       0.000000  
precision    0.000000  
sensitivity  0.000000  
specificity  0.000000  
accuracy     0.000000  
F1           0.000000  
brier        0.001766  
AUC          0.000000  
                    0         1         2         3         4      Mean  \
recall       0.636364  0.818182  0.818182  0.909091  0.700000  0.776364   
precision    0.777778  0.692308  0.642857  0.769231  0.700000  0.716435   
sensitivity  0.636364  0.818182  0.818182  0.909091  0.700000  0.776364   
specificity  0.857143  0.714286  0.642857  0.785714  0.785714  0.757143   
accuracy     0.760000  0.760000  0.720000  0.840000  0.750000  0.766000   
F1           0.700000  0.750000  0.720000  0.833333  0.700000  0.740667   
brier        0.200260  0.176984  0.207364  0.139160  0.211275  0.187009   
AUC          0.737013  0.827922  0.724026  0.873377  0.728571  0.778182   

                  Std  
recall       0.107872  
precision    0.056604  
sensitivity  0.107872  
specificity  0.081441  
accuracy     0.044497  
F1           0.055698  
brier        0.029870  
AUC          0.068237  
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------all------
              precision    recall  f1-score   support

         Neg       0.84      0.67      0.75        70
         Pos       0.66      0.83      0.74        54

   micro avg       0.74      0.74      0.74       124
   macro avg       0.75      0.75      0.74       124
weighted avg       0.76      0.74      0.74       124

[Confusion Matrix]
TN	FP
FN	TP
[[47 23]
 [ 9 45]] 

cutoff = ('Youden', 0.36)
recall = 0.8333333333333334
precision = 0.6617647058823529
sensitivity = 0.8333333333333334
specificity = 0.6714285714285714
accuracy = 0.7419354838709677
F1 = 0.7377049180327869
brier = 0.18681290322580646
AUC = 0.7804232804232805
95%CI-AUC = ('Binomial', 0.6972318653162309, 0.849762074323568)

-----Finish!-----
In [29]:
# set the structure of inputs for further feature selection
fpath_out = 'T2/Ranking-5'
if not os.path.exists(fpath_out):os.makedirs(fpath_out)
T2_output_Lasso.typeof = Further_Selection.Ranking
T2_output_Lasso.model = lr
T2_output_Lasso.name = name
T2_output_Lasso.fpath = fpath_out

# the further feature selection
T2_output_Further = Further_Selection_CV.selection(T2_output_Lasso,max_FeaNum=5)
plt.close('all')

# model developing
for clf,model_name in [(lr,'LR'),(svc,'SVM'),(rf,'RF')]:
    T2_output_Further.model = clf
    T2_output_Further.name = model_name
    T2_output_Further.fpath = fpath_out
    T2_results_CV,T2_result_all = ModelPred.OneCV(T2_output_Further,label=['Neg','Pos'])
    plt.close()
-----Start Further_Selection_CV-----
-----0-----
-----1-----
-----2-----
-----3-----
-----4-----
-----Finish!-----
-----Start Model-----
-----0-----
------test------
              precision    recall  f1-score   support

         Neg       0.87      0.93      0.90        14
         Pos       0.90      0.82      0.86        11

   micro avg       0.88      0.88      0.88        25
   macro avg       0.88      0.87      0.88        25
weighted avg       0.88      0.88      0.88        25

[Confusion Matrix]
TN	FP
FN	TP
[[13  1]
 [ 2  9]] 

cutoff = ('Youden', 0.42402832159446996)
recall = 0.8181818181818182
precision = 0.9
sensitivity = 0.8181818181818182
specificity = 0.9285714285714286
accuracy = 0.88
F1 = 0.8571428571428572
brier = 0.13747980475852128
AUC = 0.9285714285714286
95%CI-AUC = ('Binomial', 0.7513653244637356, 0.9927435353916194)

------train------
              precision    recall  f1-score   support

         Neg       0.91      0.77      0.83        56
         Pos       0.75      0.91      0.82        43

   micro avg       0.83      0.83      0.83        99
   macro avg       0.83      0.84      0.83        99
weighted avg       0.84      0.83      0.83        99

[Confusion Matrix]
TN	FP
FN	TP
[[43 13]
 [ 4 39]] 

cutoff = ('Youden', 0.4061461171253337)
recall = 0.9069767441860465
precision = 0.75
sensitivity = 0.9069767441860465
specificity = 0.7678571428571429
accuracy = 0.8282828282828283
F1 = 0.8210526315789475
brier = 0.1426153682210238
AUC = 0.8683554817275747
95%CI-AUC = ('Binomial', 0.7855335392847961, 0.9279380169555842)

-----1-----
------test------
              precision    recall  f1-score   support

         Neg       1.00      0.57      0.73        14
         Pos       0.65      1.00      0.79        11

   micro avg       0.76      0.76      0.76        25
   macro avg       0.82      0.79      0.76        25
weighted avg       0.84      0.76      0.75        25

[Confusion Matrix]
TN	FP
FN	TP
[[ 8  6]
 [ 0 11]] 

cutoff = ('Youden', 0.42278606692802445)
recall = 1.0
precision = 0.6470588235294118
sensitivity = 1.0
specificity = 0.5714285714285714
accuracy = 0.76
F1 = 0.7857142857142858
brier = 0.21611177912884738
AUC = 0.7207792207792207
95%CI-AUC = ('Binomial', 0.506938318533891, 0.8798285008906752)

------train------
              precision    recall  f1-score   support

         Neg       0.92      0.84      0.88        56
         Pos       0.81      0.91      0.86        43

   micro avg       0.87      0.87      0.87        99
   macro avg       0.87      0.87      0.87        99
weighted avg       0.87      0.87      0.87        99

[Confusion Matrix]
TN	FP
FN	TP
[[47  9]
 [ 4 39]] 

cutoff = ('Youden', 0.43483405433919586)
recall = 0.9069767441860465
precision = 0.8125
sensitivity = 0.9069767441860465
specificity = 0.8392857142857143
accuracy = 0.8686868686868687
F1 = 0.8571428571428572
brier = 0.12520390307881765
AUC = 0.9003322259136213
95%CI-AUC = ('Binomial', 0.8237075049706944, 0.9514430078931789)

-----2-----
------test------
              precision    recall  f1-score   support

         Neg       0.82      0.64      0.72        14
         Pos       0.64      0.82      0.72        11

   micro avg       0.72      0.72      0.72        25
   macro avg       0.73      0.73      0.72        25
weighted avg       0.74      0.72      0.72        25

[Confusion Matrix]
TN	FP
FN	TP
[[9 5]
 [2 9]] 

cutoff = ('Youden', 0.4408220182667645)
recall = 0.8181818181818182
precision = 0.6428571428571429
sensitivity = 0.8181818181818182
specificity = 0.6428571428571429
accuracy = 0.72
F1 = 0.7200000000000001
brier = 0.21619411310923276
AUC = 0.7142857142857143
95%CI-AUC = ('Binomial', 0.5001623351600223, 0.8752665811796557)

------train------
              precision    recall  f1-score   support

         Neg       0.92      0.80      0.86        56
         Pos       0.78      0.91      0.84        43

   micro avg       0.85      0.85      0.85        99
   macro avg       0.85      0.86      0.85        99
weighted avg       0.86      0.85      0.85        99

[Confusion Matrix]
TN	FP
FN	TP
[[45 11]
 [ 4 39]] 

cutoff = ('Youden', 0.42240251885271723)
recall = 0.9069767441860465
precision = 0.78
sensitivity = 0.9069767441860465
specificity = 0.8035714285714286
accuracy = 0.8484848484848485
F1 = 0.8387096774193548
brier = 0.12760580917330744
AUC = 0.8982558139534883
95%CI-AUC = ('Binomial', 0.8211841253135906, 0.9499661051802076)

-----3-----
------test------
              precision    recall  f1-score   support

         Neg       1.00      0.79      0.88        14
         Pos       0.79      1.00      0.88        11

   micro avg       0.88      0.88      0.88        25
   macro avg       0.89      0.89      0.88        25
weighted avg       0.91      0.88      0.88        25

[Confusion Matrix]
TN	FP
FN	TP
[[11  3]
 [ 0 11]] 

cutoff = ('Youden', 0.33555088208194583)
recall = 1.0
precision = 0.7857142857142857
sensitivity = 1.0
specificity = 0.7857142857142857
accuracy = 0.88
F1 = 0.88
brier = 0.12476560348807517
AUC = 0.9545454545454546
95%CI-AUC = ('Binomial', 0.7883419356554456, 0.9983179285128623)

------train------
              precision    recall  f1-score   support

         Neg       0.82      0.84      0.83        56
         Pos       0.79      0.77      0.78        43

   micro avg       0.81      0.81      0.81        99
   macro avg       0.81      0.80      0.80        99
weighted avg       0.81      0.81      0.81        99

[Confusion Matrix]
TN	FP
FN	TP
[[47  9]
 [10 33]] 

cutoff = ('Youden', 0.491060078753171)
recall = 0.7674418604651163
precision = 0.7857142857142857
sensitivity = 0.7674418604651163
specificity = 0.8392857142857143
accuracy = 0.8080808080808081
F1 = 0.7764705882352941
brier = 0.16721251866738704
AUC = 0.8251661129568106
95%CI-AUC = ('Binomial', 0.7358328768495677, 0.8941670790806746)

-----4-----
------test------
              precision    recall  f1-score   support

         Neg       0.89      0.57      0.70        14
         Pos       0.60      0.90      0.72        10

   micro avg       0.71      0.71      0.71        24
   macro avg       0.74      0.74      0.71        24
weighted avg       0.77      0.71      0.71        24

[Confusion Matrix]
TN	FP
FN	TP
[[8 6]
 [1 9]] 

cutoff = ('Youden', 0.22255149324338114)
recall = 0.9
precision = 0.6
sensitivity = 0.9
specificity = 0.5714285714285714
accuracy = 0.7083333333333334
F1 = 0.7200000000000001
brier = 0.21498134261464819
AUC = 0.75
95%CI-AUC = ('Binomial', 0.5328871975773306, 0.9022695905254671)

------train------
              precision    recall  f1-score   support

         Neg       0.92      0.82      0.87        56
         Pos       0.80      0.91      0.85        44

   micro avg       0.86      0.86      0.86       100
   macro avg       0.86      0.87      0.86       100
weighted avg       0.87      0.86      0.86       100

[Confusion Matrix]
TN	FP
FN	TP
[[46 10]
 [ 4 40]] 

cutoff = ('Youden', 0.4201472105200088)
recall = 0.9090909090909091
precision = 0.8
sensitivity = 0.9090909090909091
specificity = 0.8214285714285714
accuracy = 0.86
F1 = 0.8510638297872342
brier = 0.1307433166281341
AUC = 0.8944805194805194
95%CI-AUC = ('Binomial', 0.8170919016549374, 0.9470409031339475)

                    0         1         2         3         4      Mean  \
recall       0.906977  0.906977  0.906977  0.767442  0.909091  0.879493   
precision    0.750000  0.812500  0.780000  0.785714  0.800000  0.785643   
sensitivity  0.906977  0.906977  0.906977  0.767442  0.909091  0.879493   
specificity  0.767857  0.839286  0.803571  0.839286  0.821429  0.814286   
accuracy     0.828283  0.868687  0.848485  0.808081  0.860000  0.842707   
F1           0.821053  0.857143  0.838710  0.776471  0.851064  0.828888   
brier        0.142615  0.125204  0.127606  0.167213  0.130743  0.138676   
AUC          0.868355  0.900332  0.898256  0.825166  0.894481  0.877318   

                  Std  
recall       0.062645  
precision    0.023610  
sensitivity  0.062645  
specificity  0.029881  
accuracy     0.024568  
F1           0.032387  
brier        0.017298  
AUC          0.031870  
                    0         1         2         3         4      Mean  \
recall       0.818182  1.000000  0.818182  1.000000  0.900000  0.907273   
precision    0.900000  0.647059  0.642857  0.785714  0.600000  0.715126   
sensitivity  0.818182  1.000000  0.818182  1.000000  0.900000  0.907273   
specificity  0.928571  0.571429  0.642857  0.785714  0.571429  0.700000   
accuracy     0.880000  0.760000  0.720000  0.880000  0.708333  0.789667   
F1           0.857143  0.785714  0.720000  0.880000  0.720000  0.792571   
brier        0.137480  0.216112  0.216194  0.124766  0.214981  0.181907   
AUC          0.928571  0.720779  0.714286  0.954545  0.750000  0.813636   

                  Std  
recall       0.091000  
precision    0.124771  
sensitivity  0.091000  
specificity  0.154853  
accuracy     0.084659  
F1           0.074822  
brier        0.046579  
AUC          0.117907  
------all------
              precision    recall  f1-score   support

         Neg       0.88      0.71      0.79        70
         Pos       0.70      0.87      0.78        54

   micro avg       0.78      0.78      0.78       124
   macro avg       0.79      0.79      0.78       124
weighted avg       0.80      0.78      0.78       124

[Confusion Matrix]
TN	FP
FN	TP
[[50 20]
 [ 7 47]] 

cutoff = ('Youden', 0.39294575016447075)
recall = 0.8703703703703703
precision = 0.7014925373134329
sensitivity = 0.8703703703703703
specificity = 0.7142857142857143
accuracy = 0.782258064516129
F1 = 0.7768595041322315
brier = 0.1816397962489393
AUC = 0.8007936507936508
95%CI-AUC = ('Binomial', 0.7195864395976709, 0.8671080854131452)

-----Finish!-----
-----Start Model-----
-----0-----
------test------
              precision    recall  f1-score   support

         Neg       0.86      0.86      0.86        14
         Pos       0.82      0.82      0.82        11

   micro avg       0.84      0.84      0.84        25
   macro avg       0.84      0.84      0.84        25
weighted avg       0.84      0.84      0.84        25

[Confusion Matrix]
TN	FP
FN	TP
[[12  2]
 [ 2  9]] 

cutoff = ('Youden', 0.4630879768356677)
recall = 0.8181818181818182
precision = 0.8181818181818182
sensitivity = 0.8181818181818182
specificity = 0.8571428571428571
accuracy = 0.84
F1 = 0.8181818181818182
brier = 0.16489748780253388
AUC = 0.8116883116883117
95%CI-AUC = ('Binomial', 0.6062459490380403, 0.9386536573797263)

------train------
              precision    recall  f1-score   support

         Neg       0.96      0.77      0.85        56
         Pos       0.76      0.95      0.85        43

   micro avg       0.85      0.85      0.85        99
   macro avg       0.86      0.86      0.85        99
weighted avg       0.87      0.85      0.85        99

[Confusion Matrix]
TN	FP
FN	TP
[[43 13]
 [ 2 41]] 

cutoff = ('Youden', 0.2774553422787947)
recall = 0.9534883720930233
precision = 0.7592592592592593
sensitivity = 0.9534883720930233
specificity = 0.7678571428571429
accuracy = 0.8484848484848485
F1 = 0.845360824742268
brier = 0.11996114772641048
AUC = 0.8978405315614617
95%CI-AUC = ('Binomial', 0.8206802758207263, 0.9496697996819241)

-----1-----
------test------
              precision    recall  f1-score   support

         Neg       0.89      0.57      0.70        14
         Pos       0.62      0.91      0.74        11

   micro avg       0.72      0.72      0.72        25
   macro avg       0.76      0.74      0.72        25
weighted avg       0.77      0.72      0.72        25

[Confusion Matrix]
TN	FP
FN	TP
[[ 8  6]
 [ 1 10]] 

cutoff = ('Youden', 0.4800329057500271)
recall = 0.9090909090909091
precision = 0.625
sensitivity = 0.9090909090909091
specificity = 0.5714285714285714
accuracy = 0.72
F1 = 0.7407407407407406
brier = 0.2068270380367323
AUC = 0.7012987012987013
95%CI-AUC = ('Binomial', 0.4867234203121368, 0.8660179541627924)

------train------
              precision    recall  f1-score   support

         Neg       0.84      0.96      0.90        56
         Pos       0.94      0.77      0.85        43

   micro avg       0.88      0.88      0.88        99
   macro avg       0.89      0.87      0.87        99
weighted avg       0.89      0.88      0.88        99

[Confusion Matrix]
TN	FP
FN	TP
[[54  2]
 [10 33]] 

cutoff = ('Youden', 0.5885786575201262)
recall = 0.7674418604651163
precision = 0.9428571428571428
sensitivity = 0.7674418604651163
specificity = 0.9642857142857143
accuracy = 0.8787878787878788
F1 = 0.8461538461538461
brier = 0.10544300296348887
AUC = 0.9227574750830565
95%CI-AUC = ('Binomial', 0.8514499690678909, 0.9668390113927129)

-----2-----
------test------
              precision    recall  f1-score   support

         Neg       0.85      0.79      0.81        14
         Pos       0.75      0.82      0.78        11

   micro avg       0.80      0.80      0.80        25
   macro avg       0.80      0.80      0.80        25
weighted avg       0.80      0.80      0.80        25

[Confusion Matrix]
TN	FP
FN	TP
[[11  3]
 [ 2  9]] 

cutoff = ('Youden', 0.5339249544804243)
recall = 0.8181818181818182
precision = 0.75
sensitivity = 0.8181818181818182
specificity = 0.7857142857142857
accuracy = 0.8
F1 = 0.7826086956521738
brier = 0.17430344559520322
AUC = 0.8116883116883117
95%CI-AUC = ('Binomial', 0.6062459490380402, 0.9386536573797263)

------train------
              precision    recall  f1-score   support

         Neg       0.89      0.88      0.88        56
         Pos       0.84      0.86      0.85        43

   micro avg       0.87      0.87      0.87        99
   macro avg       0.87      0.87      0.87        99
weighted avg       0.87      0.87      0.87        99

[Confusion Matrix]
TN	FP
FN	TP
[[49  7]
 [ 6 37]] 

cutoff = ('Youden', 0.5)
recall = 0.8604651162790697
precision = 0.8409090909090909
sensitivity = 0.8604651162790697
specificity = 0.875
accuracy = 0.8686868686868687
F1 = 0.8505747126436781
brier = 0.12134654550691594
AUC = 0.9028239202657807
95%CI-AUC = ('Binomial', 0.8267448381442899, 0.9532048874802718)

-----3-----
------test------
              precision    recall  f1-score   support

         Neg       0.92      0.79      0.85        14
         Pos       0.77      0.91      0.83        11

   micro avg       0.84      0.84      0.84        25
   macro avg       0.84      0.85      0.84        25
weighted avg       0.85      0.84      0.84        25

[Confusion Matrix]
TN	FP
FN	TP
[[11  3]
 [ 1 10]] 

cutoff = ('Youden', 0.26034276542559837)
recall = 0.9090909090909091
precision = 0.7692307692307693
sensitivity = 0.9090909090909091
specificity = 0.7857142857142857
accuracy = 0.84
F1 = 0.8333333333333333
brier = 0.1357103317659301
AUC = 0.9285714285714285
95%CI-AUC = ('Binomial', 0.7513653244637356, 0.9927435353916194)

------train------
              precision    recall  f1-score   support

         Neg       0.80      0.88      0.84        56
         Pos       0.82      0.72      0.77        43

   micro avg       0.81      0.81      0.81        99
   macro avg       0.81      0.80      0.80        99
weighted avg       0.81      0.81      0.81        99

[Confusion Matrix]
TN	FP
FN	TP
[[49  7]
 [12 31]] 

cutoff = ('Youden', 0.5253459652307046)
recall = 0.7209302325581395
precision = 0.8157894736842105
sensitivity = 0.7209302325581395
specificity = 0.875
accuracy = 0.8080808080808081
F1 = 0.765432098765432
brier = 0.15373927863701353
AUC = 0.8666943521594684
95%CI-AUC = ('Binomial', 0.7835864224034667, 0.926677498055638)

-----4-----
------test------
              precision    recall  f1-score   support

         Neg       0.86      0.86      0.86        14
         Pos       0.80      0.80      0.80        10

   micro avg       0.83      0.83      0.83        24
   macro avg       0.83      0.83      0.83        24
weighted avg       0.83      0.83      0.83        24

[Confusion Matrix]
TN	FP
FN	TP
[[12  2]
 [ 2  8]] 

cutoff = ('Youden', 0.5242377306614807)
recall = 0.8
precision = 0.8
sensitivity = 0.8
specificity = 0.8571428571428571
accuracy = 0.8333333333333334
F1 = 0.8000000000000002
brier = 0.17083584456271447
AUC = 0.7928571428571429
95%CI-AUC = ('Binomial', 0.5798186525343071, 0.9294026470471334)

------train------
              precision    recall  f1-score   support

         Neg       0.91      0.89      0.90        56
         Pos       0.87      0.89      0.88        44

   micro avg       0.89      0.89      0.89       100
   macro avg       0.89      0.89      0.89       100
weighted avg       0.89      0.89      0.89       100

[Confusion Matrix]
TN	FP
FN	TP
[[50  6]
 [ 5 39]] 

cutoff = ('Youden', 0.44840286540223245)
recall = 0.8863636363636364
precision = 0.8666666666666667
sensitivity = 0.8863636363636364
specificity = 0.8928571428571429
accuracy = 0.89
F1 = 0.8764044943820225
brier = 0.09600555919715699
AUC = 0.9443993506493507
95%CI-AUC = ('Binomial', 0.8797335382271161, 0.9803126228456718)

                    0         1         2         3         4      Mean  \
recall       0.953488  0.767442  0.860465  0.720930  0.886364  0.837738   
precision    0.759259  0.942857  0.840909  0.815789  0.866667  0.845096   
sensitivity  0.953488  0.767442  0.860465  0.720930  0.886364  0.837738   
specificity  0.767857  0.964286  0.875000  0.875000  0.892857  0.875000   
accuracy     0.848485  0.878788  0.868687  0.808081  0.890000  0.858808   
F1           0.845361  0.846154  0.850575  0.765432  0.876404  0.836785   
brier        0.119961  0.105443  0.121347  0.153739  0.096006  0.119299   
AUC          0.897841  0.922757  0.902824  0.866694  0.944399  0.906903   

                  Std  
recall       0.093360  
precision    0.067578  
sensitivity  0.093360  
specificity  0.070304  
accuracy     0.032208  
F1           0.041870  
brier        0.021939  
AUC          0.029038  
                    0         1         2         3         4      Mean  \
recall       0.818182  0.909091  0.818182  0.909091  0.800000  0.850909   
precision    0.818182  0.625000  0.750000  0.769231  0.800000  0.752483   
sensitivity  0.818182  0.909091  0.818182  0.909091  0.800000  0.850909   
specificity  0.857143  0.571429  0.785714  0.785714  0.857143  0.771429   
accuracy     0.840000  0.720000  0.800000  0.840000  0.833333  0.806667   
F1           0.818182  0.740741  0.782609  0.833333  0.800000  0.794973   
brier        0.164897  0.206827  0.174303  0.135710  0.170836  0.170515   
AUC          0.811688  0.701299  0.811688  0.928571  0.792857  0.809221   

                  Std  
recall       0.053629  
precision    0.076014  
sensitivity  0.053629  
specificity  0.117369  
accuracy     0.051208  
F1           0.035810  
brier        0.025377  
AUC          0.080888  
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------all------
              precision    recall  f1-score   support

         Neg       0.85      0.79      0.81        70
         Pos       0.75      0.81      0.78        54

   micro avg       0.80      0.80      0.80       124
   macro avg       0.80      0.80      0.80       124
weighted avg       0.80      0.80      0.80       124

[Confusion Matrix]
TN	FP
FN	TP
[[55 15]
 [10 44]] 

cutoff = ('Youden', 0.43150875541595096)
recall = 0.8148148148148148
precision = 0.7457627118644068
sensitivity = 0.8148148148148148
specificity = 0.7857142857142857
accuracy = 0.7983870967741935
F1 = 0.7787610619469028
brier = 0.17051224072189622
AUC = 0.8115079365079365
95%CI-AUC = ('Binomial', 0.7314520381363178, 0.8761194755694358)

-----Finish!-----
-----Start Model-----
-----0-----
------test------
              precision    recall  f1-score   support

         Neg       0.83      0.71      0.77        14
         Pos       0.69      0.82      0.75        11

   micro avg       0.76      0.76      0.76        25
   macro avg       0.76      0.77      0.76        25
weighted avg       0.77      0.76      0.76        25

[Confusion Matrix]
TN	FP
FN	TP
[[10  4]
 [ 2  9]] 

cutoff = ('Youden', 0.45)
recall = 0.8181818181818182
precision = 0.6923076923076923
sensitivity = 0.8181818181818182
specificity = 0.7142857142857143
accuracy = 0.76
F1 = 0.7500000000000001
brier = 0.204808
AUC = 0.7402597402597403
95%CI-AUC = ('Binomial', 0.5275004861125896, 0.8932535525360982)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

         Neg       1.00      1.00      1.00        56
         Pos       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.56)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.02576767676767676
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

-----1-----
------test------
              precision    recall  f1-score   support

         Neg       1.00      0.57      0.73        14
         Pos       0.65      1.00      0.79        11

   micro avg       0.76      0.76      0.76        25
   macro avg       0.82      0.79      0.76        25
weighted avg       0.84      0.76      0.75        25

[Confusion Matrix]
TN	FP
FN	TP
[[ 8  6]
 [ 0 11]] 

cutoff = ('Youden', 0.3)
recall = 1.0
precision = 0.6470588235294118
sensitivity = 1.0
specificity = 0.5714285714285714
accuracy = 0.76
F1 = 0.7857142857142858
brier = 0.16261199999999998
AUC = 0.8506493506493505
95%CI-AUC = ('Binomial', 0.6518580242477625, 0.960255571076826)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

         Neg       1.00      1.00      1.00        56
         Pos       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.5)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.028577777777777785
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

-----2-----
------test------
              precision    recall  f1-score   support

         Neg       0.77      0.71      0.74        14
         Pos       0.67      0.73      0.70        11

   micro avg       0.72      0.72      0.72        25
   macro avg       0.72      0.72      0.72        25
weighted avg       0.72      0.72      0.72        25

[Confusion Matrix]
TN	FP
FN	TP
[[10  4]
 [ 3  8]] 

cutoff = ('Youden', 0.5)
recall = 0.7272727272727273
precision = 0.6666666666666666
sensitivity = 0.7272727272727273
specificity = 0.7142857142857143
accuracy = 0.72
F1 = 0.6956521739130435
brier = 0.20953600000000003
AUC = 0.737012987012987
95%CI-AUC = ('Binomial', 0.524048524156825, 0.8910439384661436)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

         Neg       1.00      1.00      1.00        56
         Pos       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.58)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.024962626262626266
AUC = 0.9999999999999999
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

-----3-----
------test------
              precision    recall  f1-score   support

         Neg       0.86      0.86      0.86        14
         Pos       0.82      0.82      0.82        11

   micro avg       0.84      0.84      0.84        25
   macro avg       0.84      0.84      0.84        25
weighted avg       0.84      0.84      0.84        25

[Confusion Matrix]
TN	FP
FN	TP
[[12  2]
 [ 2  9]] 

cutoff = ('Youden', 0.4)
recall = 0.8181818181818182
precision = 0.8181818181818182
sensitivity = 0.8181818181818182
specificity = 0.8571428571428571
accuracy = 0.84
F1 = 0.8181818181818182
brier = 0.15539199999999997
AUC = 0.8701298701298702
95%CI-AUC = ('Binomial', 0.6755464765688096, 0.9699559155493546)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

         Neg       1.00      1.00      1.00        56
         Pos       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.63)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.03036868686868688
AUC = 0.9999999999999999
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

-----4-----
------test------
              precision    recall  f1-score   support

         Neg       0.86      0.86      0.86        14
         Pos       0.80      0.80      0.80        10

   micro avg       0.83      0.83      0.83        24
   macro avg       0.83      0.83      0.83        24
weighted avg       0.83      0.83      0.83        24

[Confusion Matrix]
TN	FP
FN	TP
[[12  2]
 [ 2  8]] 

cutoff = ('Youden', 0.55)
recall = 0.8
precision = 0.8
sensitivity = 0.8
specificity = 0.8571428571428571
accuracy = 0.8333333333333334
F1 = 0.8000000000000002
brier = 0.18628749999999997
AUC = 0.775
95%CI-AUC = ('Binomial', 0.5600187574967712, 0.918381272983291)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

         Neg       1.00      1.00      1.00        56
         Pos       1.00      1.00      1.00        44

   micro avg       1.00      1.00      1.00       100
   macro avg       1.00      1.00      1.00       100
weighted avg       1.00      1.00      1.00       100

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 44]] 

cutoff = ('Youden', 0.61)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.024510999999999995
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9637833073548235, nan)

                    0         1         2         3         4      Mean  \
recall       1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
precision    1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
sensitivity  1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
specificity  1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
accuracy     1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
F1           1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
brier        0.025768  0.028578  0.024963  0.030369  0.024511  0.026838   
AUC          1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   

                      Std  
recall       0.000000e+00  
precision    0.000000e+00  
sensitivity  0.000000e+00  
specificity  0.000000e+00  
accuracy     0.000000e+00  
F1           0.000000e+00  
brier        2.528345e-03  
AUC          7.850462e-17  
                    0         1         2         3         4      Mean  \
recall       0.818182  1.000000  0.727273  0.818182  0.800000  0.832727   
precision    0.692308  0.647059  0.666667  0.818182  0.800000  0.724843   
sensitivity  0.818182  1.000000  0.727273  0.818182  0.800000  0.832727   
specificity  0.714286  0.571429  0.714286  0.857143  0.857143  0.742857   
accuracy     0.760000  0.760000  0.720000  0.840000  0.833333  0.782667   
F1           0.750000  0.785714  0.695652  0.818182  0.800000  0.769910   
brier        0.204808  0.162612  0.209536  0.155392  0.186287  0.183727   
AUC          0.740260  0.850649  0.737013  0.870130  0.775000  0.794610   

                  Std  
recall       0.100741  
precision    0.078826  
sensitivity  0.100741  
specificity  0.119523  
accuracy     0.051983  
F1           0.048466  
brier        0.024320  
AUC          0.062249  
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------all------
              precision    recall  f1-score   support

         Neg       0.80      0.79      0.79        70
         Pos       0.73      0.74      0.73        54

   micro avg       0.77      0.77      0.77       124
   macro avg       0.76      0.76      0.76       124
weighted avg       0.77      0.77      0.77       124

[Confusion Matrix]
TN	FP
FN	TP
[[55 15]
 [14 40]] 

cutoff = ('Youden', 0.5)
recall = 0.7407407407407407
precision = 0.7272727272727273
sensitivity = 0.7407407407407407
specificity = 0.7857142857142857
accuracy = 0.7661290322580645
F1 = 0.7339449541284404
brier = 0.18370645161290325
AUC = 0.7906084656084656
95%CI-AUC = ('Binomial', 0.7083768285246775, 0.8584686859802543)

-----Finish!-----
In [30]:
# set the name of dirs and names
fpath_out = 'TC'
name = 'TC'

# judge, read and standardize the X 
if 'TC' in files:
    X_TC_Sd = read_csv('TC/X_TC_Sd.csv',header=0,index_col=0)
else:
    if not os.path.exists(fpath_out):os.makedirs(fpath_out)
    X_TC = read_csv('X_all_TC.csv',header=0,index_col=0)
    # X_TC_Sd = Std_features(X_TC)
    # X_TC_Sd.to_csv('TC/X_TC_Sd.csv')
    
# set the structure of inputs
fpath_out = 'TC/LASSO'
if not os.path.exists(fpath_out):os.makedirs(fpath_out)
TC_input = paras(X=X_TC,y=y_all,ID_test=ID_test,ID_train=ID_train,name=name,fpath=fpath_out)

# the first step of feature selection 
TC_output_Lasso = LASSO_CV.selection(TC_input,inner_n_splits=10)
plt.close('all')

# model developing
for clf,model_name in [(lr,'LR'),(svc,'SVM'),(rf,'RF')]:
    TC_output_Lasso.model = clf
    TC_output_Lasso.name = model_name
    TC_output_Lasso.fpath = fpath_out
    TC_results_CV,TC_result_all = ModelPred.OneCV(TC_output_Lasso,label=['NC','RB'])
    plt.close()
-----Start LASSO_CV-----
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
Alpha: 0.0360523386542392

Features reduced from 1029 to 25 by LASSO
                                                   Coefficients
Features                                                       
original_shape_Compactness2                           -0.107429
wavelet-LHL_glcm_InverseVariance                      -0.071882
wavelet-LHH_firstorder_Kurtosis                       -0.068255
wavelet-LHH_firstorder_Mean                           -0.034603
exponential_glszm_SmallAreaLowGrayLevelEmphasis       -0.030890
squareroot_glrlm_LongRunLowGrayLevelEmphasis          -0.020861
wavelet-HHL_firstorder_Kurtosis                       -0.015594
exponential_glszm_SmallAreaEmphasis                   -0.013862
wavelet-HLL_glszm_SizeZoneNonUniformityNormalized     -0.013301
wavelet-HHL_glcm_MaximumProbability                   -0.010520
wavelet-HLH_firstorder_Median                         -0.010065
wavelet-HHH_firstorder_Skewness                       -0.008420
wavelet-LHH_glcm_ClusterShade                         -0.001392
wavelet-LHH_firstorder_Skewness                       -0.001170
original_shape_Elongation                              0.007603
wavelet-HLL_firstorder_Mean                            0.007973
wavelet-LLH_glcm_Imc1                                  0.013875
exponential_firstorder_Minimum                         0.022174
wavelet-HHL_glcm_InverseVariance                       0.027059
wavelet-LLH_firstorder_Skewness                        0.028777
wavelet-HHH_glcm_Autocorrelation                       0.029876
wavelet-HHL_glszm_GrayLevelNonUniformity               0.050246
exponential_firstorder_Kurtosis                        0.051480
original_shape_LeastAxis                               0.096563
logarithm_glrlm_GrayLevelNonUniformity                 0.130806
-----0-----
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
Alpha: 0.04856092657583446
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
Features reduced from 1029 to 19 by LASSO
                                                   Coefficients
Features                                                       
wavelet-LHL_glcm_InverseVariance                      -0.086180
original_shape_Compactness2                           -0.040668
original_shape_Compactness1                           -0.034836
wavelet-LHH_glcm_Correlation                          -0.032947
wavelet-HLL_glszm_SizeZoneNonUniformityNormalized     -0.028016
wavelet-HHL_glcm_MaximumProbability                   -0.021003
wavelet-LHH_firstorder_Kurtosis                       -0.018609
wavelet-HHH_firstorder_Skewness                       -0.016327
wavelet-HHL_firstorder_Kurtosis                       -0.015393
exponential_glszm_SmallAreaHighGrayLevelEmphasis      -0.010000
wavelet-LHH_firstorder_Skewness                       -0.006316
squareroot_glrlm_LongRunLowGrayLevelEmphasis          -0.000322
square_glcm_Imc1                                       0.000115
wavelet-HLL_glcm_Imc1                                  0.004481
wavelet-HLH_firstorder_Mean                            0.008077
logarithm_glszm_GrayLevelNonUniformity                 0.031450
wavelet-LLH_firstorder_Skewness                        0.034260
logarithm_glrlm_GrayLevelNonUniformity                 0.101044
original_shape_SurfaceArea                             0.112871
-----1-----
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
Alpha: 0.03670206847993424
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
Features reduced from 1029 to 27 by LASSO
                                                   Coefficients
Features                                                       
original_shape_Compactness2                           -0.080054
wavelet-LHL_glcm_InverseVariance                      -0.060214
wavelet-HHH_firstorder_Skewness                       -0.033800
exponential_glszm_SmallAreaLowGrayLevelEmphasis       -0.030710
wavelet-HLL_glszm_SizeZoneNonUniformityNormalized     -0.026523
wavelet-LHH_glcm_ClusterShade                         -0.022398
exponential_glszm_SmallAreaEmphasis                   -0.022174
wavelet-HLL_firstorder_Kurtosis                       -0.021010
wavelet-LLH_firstorder_Median                         -0.011947
exponential_glcm_Correlation                          -0.011513
wavelet-LHH_firstorder_Skewness                       -0.007238
exponential_glszm_ZonePercentage                      -0.001331
wavelet-LHL_glszm_LargeAreaLowGrayLevelEmphasis        0.000271
wavelet-HLH_glrlm_ShortRunLowGrayLevelEmphasis         0.007248
logarithm_glszm_GrayLevelNonUniformity                 0.008632
wavelet-LLH_glcm_Imc1                                  0.009854
wavelet-HLL_firstorder_Mean                            0.015160
exponential_firstorder_Minimum                         0.015915
wavelet-LHH_glszm_SizeZoneNonUniformity                0.015918
wavelet-LLH_glrlm_ShortRunLowGrayLevelEmphasis         0.016950
original_shape_Maximum2DDiameterRow                    0.018271
wavelet-LLH_firstorder_Skewness                        0.020378
wavelet-HLL_glrlm_ShortRunLowGrayLevelEmphasis         0.036538
square_glcm_InverseVariance                            0.042734
wavelet-HHH_glszm_SizeZoneNonUniformity                0.069047
wavelet-HHH_firstorder_TotalEnergy                     0.084055
logarithm_glrlm_GrayLevelNonUniformity                 0.129956
-----2-----
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
Alpha: 0.0746381877770226
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
Features reduced from 1029 to 8 by LASSO
                                          Coefficients
Features                                              
wavelet-LHL_glcm_InverseVariance             -0.061716
original_shape_Compactness2                  -0.014791
wavelet-HHH_firstorder_Skewness              -0.000515
wavelet-HHH_glszm_GrayLevelNonUniformity      0.008518
wavelet-LLH_firstorder_Skewness               0.014395
original_shape_Maximum2DDiameterRow           0.029537
original_shape_SurfaceArea                    0.071065
logarithm_glrlm_GrayLevelNonUniformity        0.082379
-----3-----
Alpha: 0.054333567480365094
/home/huiying/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.
  ConvergenceWarning)
Features reduced from 1029 to 16 by LASSO
                                                   Coefficients
Features                                                       
original_shape_Compactness2                           -0.072087
exponential_glszm_SmallAreaEmphasis                   -0.027201
wavelet-HHH_firstorder_Skewness                       -0.015654
exponential_glszm_SmallAreaLowGrayLevelEmphasis       -0.008773
wavelet-HHL_glszm_SizeZoneNonUniformityNormalized     -0.007408
wavelet-LHL_firstorder_Skewness                       -0.006026
squareroot_glrlm_LongRunLowGrayLevelEmphasis          -0.003764
exponential_glszm_SmallAreaHighGrayLevelEmphasis      -0.003733
wavelet-HLL_glcm_Imc1                                  0.000757
wavelet-LLH_firstorder_Skewness                        0.006588
squareroot_glszm_LargeAreaHighGrayLevelEmphasis        0.007979
wavelet-HHH_glszm_SizeZoneNonUniformity                0.020410
wavelet-HHL_glszm_GrayLevelNonUniformity               0.028253
original_shape_Maximum2DDiameterRow                    0.051761
original_shape_SurfaceArea                             0.062984
logarithm_glrlm_GrayLevelNonUniformity                 0.093593
-----4-----
-----Finish!-----
-----Start Model-----
-----0-----
------test------
              precision    recall  f1-score   support

          NC       0.75      0.86      0.80        14
          RB       0.78      0.64      0.70        11

   micro avg       0.76      0.76      0.76        25
   macro avg       0.76      0.75      0.75        25
weighted avg       0.76      0.76      0.76        25

[Confusion Matrix]
TN	FP
FN	TP
[[12  2]
 [ 4  7]] 

cutoff = ('Youden', 0.6821680243603379)
recall = 0.6363636363636364
precision = 0.7777777777777778
sensitivity = 0.6363636363636364
specificity = 0.8571428571428571
accuracy = 0.76
F1 = 0.7000000000000001
brier = 0.22859117109200205
AUC = 0.7402597402597402
95%CI-AUC = ('Binomial', 0.5275004861125896, 0.8932535525360982)

------train------
              precision    recall  f1-score   support

          NC       0.98      0.93      0.95        56
          RB       0.91      0.98      0.94        43

   micro avg       0.95      0.95      0.95        99
   macro avg       0.95      0.95      0.95        99
weighted avg       0.95      0.95      0.95        99

[Confusion Matrix]
TN	FP
FN	TP
[[52  4]
 [ 1 42]] 

cutoff = ('Youden', 0.4474679752341732)
recall = 0.9767441860465116
precision = 0.9130434782608695
sensitivity = 0.9767441860465116
specificity = 0.9285714285714286
accuracy = 0.9494949494949495
F1 = 0.9438202247191011
brier = 0.06449347517994379
AUC = 0.9754983388704319
95%CI-AUC = ('Binomial', 0.922459034991654, 0.9960625121752468)

-----1-----
------test------
              precision    recall  f1-score   support

          NC       0.89      0.57      0.70        14
          RB       0.62      0.91      0.74        11

   micro avg       0.72      0.72      0.72        25
   macro avg       0.76      0.74      0.72        25
weighted avg       0.77      0.72      0.72        25

[Confusion Matrix]
TN	FP
FN	TP
[[ 8  6]
 [ 1 10]] 

cutoff = ('Youden', 0.4637770990546334)
recall = 0.9090909090909091
precision = 0.625
sensitivity = 0.9090909090909091
specificity = 0.5714285714285714
accuracy = 0.72
F1 = 0.7407407407407406
brier = 0.2797903889987711
AUC = 0.6688311688311688
95%CI-AUC = ('Binomial', 0.4537589122433224, 0.8422050155666465)

------train------
              precision    recall  f1-score   support

          NC       0.89      0.96      0.92        56
          RB       0.95      0.84      0.89        43

   micro avg       0.91      0.91      0.91        99
   macro avg       0.92      0.90      0.91        99
weighted avg       0.91      0.91      0.91        99

[Confusion Matrix]
TN	FP
FN	TP
[[54  2]
 [ 7 36]] 

cutoff = ('Youden', 0.6374783907804753)
recall = 0.8372093023255814
precision = 0.9473684210526315
sensitivity = 0.8372093023255814
specificity = 0.9642857142857143
accuracy = 0.9090909090909091
F1 = 0.8888888888888888
brier = 0.07214782964776685
AUC = 0.9717607973421927
95%CI-AUC = ('Binomial', 0.9169725085623216, 0.9945857476254419)

-----2-----
------test------
              precision    recall  f1-score   support

          NC       0.65      0.93      0.76        14
          RB       0.80      0.36      0.50        11

   micro avg       0.68      0.68      0.68        25
   macro avg       0.73      0.65      0.63        25
weighted avg       0.72      0.68      0.65        25

[Confusion Matrix]
TN	FP
FN	TP
[[13  1]
 [ 7  4]] 

cutoff = ('Youden', 0.9982237229853242)
recall = 0.36363636363636365
precision = 0.8
sensitivity = 0.36363636363636365
specificity = 0.9285714285714286
accuracy = 0.68
F1 = 0.5000000000000001
brier = 0.33880970084899076
AUC = 0.6233766233766234
95%CI-AUC = ('Binomial', 0.40903781261859157, 0.8073306742718196)

------train------
              precision    recall  f1-score   support

          NC       0.96      0.96      0.96        56
          RB       0.95      0.95      0.95        43

   micro avg       0.96      0.96      0.96        99
   macro avg       0.96      0.96      0.96        99
weighted avg       0.96      0.96      0.96        99

[Confusion Matrix]
TN	FP
FN	TP
[[54  2]
 [ 2 41]] 

cutoff = ('Youden', 0.5236519471783433)
recall = 0.9534883720930233
precision = 0.9534883720930233
sensitivity = 0.9534883720930233
specificity = 0.9642857142857143
accuracy = 0.9595959595959596
F1 = 0.9534883720930233
brier = 0.04856548130851752
AUC = 0.9883720930232558
95%CI-AUC = ('Binomial', 0.9424691550036365, 0.9995547707563381)

-----3-----
------test------
              precision    recall  f1-score   support

          NC       0.86      0.86      0.86        14
          RB       0.82      0.82      0.82        11

   micro avg       0.84      0.84      0.84        25
   macro avg       0.84      0.84      0.84        25
weighted avg       0.84      0.84      0.84        25

[Confusion Matrix]
TN	FP
FN	TP
[[12  2]
 [ 2  9]] 

cutoff = ('Youden', 0.599799407845647)
recall = 0.8181818181818182
precision = 0.8181818181818182
sensitivity = 0.8181818181818182
specificity = 0.8571428571428571
accuracy = 0.84
F1 = 0.8181818181818182
brier = 0.14674895879763852
AUC = 0.8441558441558441
95%CI-AUC = ('Binomial', 0.6441014717388924, 0.9568456190770979)

------train------
              precision    recall  f1-score   support

          NC       0.81      0.86      0.83        56
          RB       0.80      0.74      0.77        43

   micro avg       0.81      0.81      0.81        99
   macro avg       0.81      0.80      0.80        99
weighted avg       0.81      0.81      0.81        99

[Confusion Matrix]
TN	FP
FN	TP
[[48  8]
 [11 32]] 

cutoff = ('Youden', 0.4828510738214207)
recall = 0.7441860465116279
precision = 0.8
sensitivity = 0.7441860465116279
specificity = 0.8571428571428571
accuracy = 0.8080808080808081
F1 = 0.7710843373493975
brier = 0.14429198120010017
AUC = 0.8745847176079734
95%CI-AUC = ('Binomial', 0.7928642047132081, 0.9326333773965327)

-----4-----
------test------
              precision    recall  f1-score   support

          NC       0.75      0.86      0.80        14
          RB       0.75      0.60      0.67        10

   micro avg       0.75      0.75      0.75        24
   macro avg       0.75      0.73      0.73        24
weighted avg       0.75      0.75      0.74        24

[Confusion Matrix]
TN	FP
FN	TP
[[12  2]
 [ 4  6]] 

cutoff = ('Youden', 0.5423370569996376)
recall = 0.6
precision = 0.75
sensitivity = 0.6
specificity = 0.8571428571428571
accuracy = 0.75
F1 = 0.6666666666666665
brier = 0.19652326626678018
AUC = 0.7785714285714286
95%CI-AUC = ('Binomial', 0.5639496858094944, 0.920619488150788)

------train------
              precision    recall  f1-score   support

          NC       0.90      0.80      0.85        56
          RB       0.78      0.89      0.83        44

   micro avg       0.84      0.84      0.84       100
   macro avg       0.84      0.84      0.84       100
weighted avg       0.85      0.84      0.84       100

[Confusion Matrix]
TN	FP
FN	TP
[[45 11]
 [ 5 39]] 

cutoff = ('Youden', 0.40576015564975926)
recall = 0.8863636363636364
precision = 0.78
sensitivity = 0.8863636363636364
specificity = 0.8035714285714286
accuracy = 0.84
F1 = 0.8297872340425532
brier = 0.11695831478320162
AUC = 0.9188311688311688
95%CI-AUC = ('Binomial', 0.8469797576538746, 0.9640441050455238)

                    0         1         2         3         4      Mean  \
recall       0.976744  0.837209  0.953488  0.744186  0.886364  0.879598   
precision    0.913043  0.947368  0.953488  0.800000  0.780000  0.878780   
sensitivity  0.976744  0.837209  0.953488  0.744186  0.886364  0.879598   
specificity  0.928571  0.964286  0.964286  0.857143  0.803571  0.903571   
accuracy     0.949495  0.909091  0.959596  0.808081  0.840000  0.893253   
F1           0.943820  0.888889  0.953488  0.771084  0.829787  0.877414   
brier        0.064493  0.072148  0.048565  0.144292  0.116958  0.089291   
AUC          0.975498  0.971761  0.988372  0.874585  0.918831  0.945809   

                  Std  
recall       0.093643  
precision    0.082800  
sensitivity  0.093643  
specificity  0.070981  
accuracy     0.066906  
F1           0.077303  
brier        0.039868  
AUC          0.047873  
                    0         1         2         3         4      Mean  \
recall       0.636364  0.909091  0.363636  0.818182  0.600000  0.665455   
precision    0.777778  0.625000  0.800000  0.818182  0.750000  0.754192   
sensitivity  0.636364  0.909091  0.363636  0.818182  0.600000  0.665455   
specificity  0.857143  0.571429  0.928571  0.857143  0.857143  0.814286   
accuracy     0.760000  0.720000  0.680000  0.840000  0.750000  0.750000   
F1           0.700000  0.740741  0.500000  0.818182  0.666667  0.685118   
brier        0.228591  0.279790  0.338810  0.146749  0.196523  0.238093   
AUC          0.740260  0.668831  0.623377  0.844156  0.778571  0.731039   

                  Std  
recall       0.211488  
precision    0.076579  
sensitivity  0.211488  
specificity  0.139240  
accuracy     0.059161  
F1           0.117931  
brier        0.074237  
AUC          0.087465  
------all------
              precision    recall  f1-score   support

          NC       0.76      0.69      0.72        70
          RB       0.64      0.72      0.68        54

   micro avg       0.70      0.70      0.70       124
   macro avg       0.70      0.70      0.70       124
weighted avg       0.71      0.70      0.70       124

[Confusion Matrix]
TN	FP
FN	TP
[[48 22]
 [15 39]] 

cutoff = ('Youden', 0.49102917595417156)
recall = 0.7222222222222222
precision = 0.639344262295082
sensitivity = 0.7222222222222222
specificity = 0.6857142857142857
accuracy = 0.7016129032258065
F1 = 0.6782608695652174
brier = 0.2384279345470789
AUC = 0.7227513227513228
95%CI-AUC = ('Binomial', 0.6352111815871254, 0.7993399038786299)

-----Finish!-----
-----Start Model-----
-----0-----
------test------
              precision    recall  f1-score   support

          NC       0.76      0.93      0.84        14
          RB       0.88      0.64      0.74        11

   micro avg       0.80      0.80      0.80        25
   macro avg       0.82      0.78      0.79        25
weighted avg       0.81      0.80      0.79        25

[Confusion Matrix]
TN	FP
FN	TP
[[13  1]
 [ 4  7]] 

cutoff = ('Youden', 0.7526627528175824)
recall = 0.6363636363636364
precision = 0.875
sensitivity = 0.6363636363636364
specificity = 0.9285714285714286
accuracy = 0.8
F1 = 0.7368421052631579
brier = 0.21298718003775144
AUC = 0.7532467532467533
95%CI-AUC = ('Binomial', 0.5414114365965182, 0.9019757858943704)

------train------
              precision    recall  f1-score   support

          NC       0.98      0.93      0.95        56
          RB       0.91      0.98      0.94        43

   micro avg       0.95      0.95      0.95        99
   macro avg       0.95      0.95      0.95        99
weighted avg       0.95      0.95      0.95        99

[Confusion Matrix]
TN	FP
FN	TP
[[52  4]
 [ 1 42]] 

cutoff = ('Youden', 0.42289476078803107)
recall = 0.9767441860465116
precision = 0.9130434782608695
sensitivity = 0.9767441860465116
specificity = 0.9285714285714286
accuracy = 0.9494949494949495
F1 = 0.9438202247191011
brier = 0.05113424740709731
AUC = 0.9842192691029901
95%CI-AUC = ('Binomial', 0.9357811252102437, 0.998764025229285)

-----1-----
------test------
              precision    recall  f1-score   support

          NC       0.89      0.57      0.70        14
          RB       0.62      0.91      0.74        11

   micro avg       0.72      0.72      0.72        25
   macro avg       0.76      0.74      0.72        25
weighted avg       0.77      0.72      0.72        25

[Confusion Matrix]
TN	FP
FN	TP
[[ 8  6]
 [ 1 10]] 

cutoff = ('Youden', 0.33078777850598057)
recall = 0.9090909090909091
precision = 0.625
sensitivity = 0.9090909090909091
specificity = 0.5714285714285714
accuracy = 0.72
F1 = 0.7407407407407406
brier = 0.2611122038633015
AUC = 0.6493506493506493
95%CI-AUC = ('Binomial', 0.43439476380998987, 0.8274697272569396)

------train------
              precision    recall  f1-score   support

          NC       0.95      0.95      0.95        56
          RB       0.93      0.93      0.93        43

   micro avg       0.94      0.94      0.94        99
   macro avg       0.94      0.94      0.94        99
weighted avg       0.94      0.94      0.94        99

[Confusion Matrix]
TN	FP
FN	TP
[[53  3]
 [ 3 40]] 

cutoff = ('Youden', 0.4130642527481867)
recall = 0.9302325581395349
precision = 0.9302325581395349
sensitivity = 0.9302325581395349
specificity = 0.9464285714285714
accuracy = 0.9393939393939394
F1 = 0.9302325581395349
brier = 0.053878309298437684
AUC = 0.9842192691029901
95%CI-AUC = ('Binomial', 0.9357811252102438, 0.998764025229285)

-----2-----
------test------
              precision    recall  f1-score   support

          NC       0.86      0.43      0.57        14
          RB       0.56      0.91      0.69        11

   micro avg       0.64      0.64      0.64        25
   macro avg       0.71      0.67      0.63        25
weighted avg       0.72      0.64      0.62        25

[Confusion Matrix]
TN	FP
FN	TP
[[ 6  8]
 [ 1 10]] 

cutoff = ('Youden', 0.3393315741265286)
recall = 0.9090909090909091
precision = 0.5555555555555556
sensitivity = 0.9090909090909091
specificity = 0.42857142857142855
accuracy = 0.64
F1 = 0.6896551724137931
brier = 0.2661181776241424
AUC = 0.6103896103896104
95%CI-AUC = ('Binomial', 0.39655178479998426, 0.7970580784279642)

------train------
              precision    recall  f1-score   support

          NC       0.98      0.96      0.97        56
          RB       0.95      0.98      0.97        43

   micro avg       0.97      0.97      0.97        99
   macro avg       0.97      0.97      0.97        99
weighted avg       0.97      0.97      0.97        99

[Confusion Matrix]
TN	FP
FN	TP
[[54  2]
 [ 1 42]] 

cutoff = ('Youden', 0.4579476842038234)
recall = 0.9767441860465116
precision = 0.9545454545454546
sensitivity = 0.9767441860465116
specificity = 0.9642857142857143
accuracy = 0.9696969696969697
F1 = 0.9655172413793104
brier = 0.039957808339794815
AUC = 0.9933554817275748
95%CI-AUC = ('Binomial', 0.9509283035284491, 0.9999683493757263)

-----3-----
------test------
              precision    recall  f1-score   support

          NC       0.92      0.86      0.89        14
          RB       0.83      0.91      0.87        11

   micro avg       0.88      0.88      0.88        25
   macro avg       0.88      0.88      0.88        25
weighted avg       0.88      0.88      0.88        25

[Confusion Matrix]
TN	FP
FN	TP
[[12  2]
 [ 1 10]] 

cutoff = ('Youden', 0.486970763318792)
recall = 0.9090909090909091
precision = 0.8333333333333334
sensitivity = 0.9090909090909091
specificity = 0.8571428571428571
accuracy = 0.88
F1 = 0.8695652173913043
brier = 0.1413554023992835
AUC = 0.9025974025974025
95%CI-AUC = ('Binomial', 0.716645614645395, 0.9840107496477167)

------train------
              precision    recall  f1-score   support

          NC       0.88      0.91      0.89        56
          RB       0.88      0.84      0.86        43

   micro avg       0.88      0.88      0.88        99
   macro avg       0.88      0.87      0.88        99
weighted avg       0.88      0.88      0.88        99

[Confusion Matrix]
TN	FP
FN	TP
[[51  5]
 [ 7 36]] 

cutoff = ('Youden', 0.46817051105222773)
recall = 0.8372093023255814
precision = 0.8780487804878049
sensitivity = 0.8372093023255814
specificity = 0.9107142857142857
accuracy = 0.8787878787878788
F1 = 0.8571428571428572
brier = 0.12356101861703364
AUC = 0.9065614617940199
95%CI-AUC = ('Binomial', 0.8313204847049749, 0.9558256014013052)

-----4-----
------test------
              precision    recall  f1-score   support

          NC       0.79      0.79      0.79        14
          RB       0.70      0.70      0.70        10

   micro avg       0.75      0.75      0.75        24
   macro avg       0.74      0.74      0.74        24
weighted avg       0.75      0.75      0.75        24

[Confusion Matrix]
TN	FP
FN	TP
[[11  3]
 [ 3  7]] 

cutoff = ('Youden', 0.5)
recall = 0.7
precision = 0.7
sensitivity = 0.7
specificity = 0.7857142857142857
accuracy = 0.75
F1 = 0.7
brier = 0.19791343338767742
AUC = 0.75
95%CI-AUC = ('Binomial', 0.5328871975773306, 0.9022695905254671)

------train------
              precision    recall  f1-score   support

          NC       0.94      0.84      0.89        56
          RB       0.82      0.93      0.87        44

   micro avg       0.88      0.88      0.88       100
   macro avg       0.88      0.89      0.88       100
weighted avg       0.89      0.88      0.88       100

[Confusion Matrix]
TN	FP
FN	TP
[[47  9]
 [ 3 41]] 

cutoff = ('Youden', 0.4221527777010454)
recall = 0.9318181818181818
precision = 0.82
sensitivity = 0.9318181818181818
specificity = 0.8392857142857143
accuracy = 0.88
F1 = 0.8723404255319149
brier = 0.10140142182316429
AUC = 0.953327922077922
95%CI-AUC = ('Binomial', 0.891636369344168, 0.9854339815642317)

                    0         1         2         3         4      Mean  \
recall       0.976744  0.930233  0.976744  0.837209  0.931818  0.930550   
precision    0.913043  0.930233  0.954545  0.878049  0.820000  0.899174   
sensitivity  0.976744  0.930233  0.976744  0.837209  0.931818  0.930550   
specificity  0.928571  0.946429  0.964286  0.910714  0.839286  0.917857   
accuracy     0.949495  0.939394  0.969697  0.878788  0.880000  0.923475   
F1           0.943820  0.930233  0.965517  0.857143  0.872340  0.913811   
brier        0.051134  0.053878  0.039958  0.123561  0.101401  0.073987   
AUC          0.984219  0.984219  0.993355  0.906561  0.953328  0.964337   

                  Std  
recall       0.056969  
precision    0.052292  
sensitivity  0.056969  
specificity  0.048247  
accuracy     0.041695  
F1           0.046837  
brier        0.036379  
AUC          0.035679  
                    0         1         2         3         4      Mean  \
recall       0.636364  0.909091  0.909091  0.909091  0.700000  0.812727   
precision    0.875000  0.625000  0.555556  0.833333  0.700000  0.717778   
sensitivity  0.636364  0.909091  0.909091  0.909091  0.700000  0.812727   
specificity  0.928571  0.571429  0.428571  0.857143  0.785714  0.714286   
accuracy     0.800000  0.720000  0.640000  0.880000  0.750000  0.758000   
F1           0.736842  0.740741  0.689655  0.869565  0.700000  0.747361   
brier        0.212987  0.261112  0.266118  0.141355  0.197913  0.215897   
AUC          0.753247  0.649351  0.610390  0.902597  0.750000  0.733117   

                  Std  
recall       0.133856  
precision    0.135381  
sensitivity  0.133856  
specificity  0.208248  
accuracy     0.089554  
F1           0.071870  
brier        0.051125  
AUC          0.113460  
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------all------
              precision    recall  f1-score   support

          NC       0.79      0.70      0.74        70
          RB       0.66      0.76      0.71        54

   micro avg       0.73      0.73      0.73       124
   macro avg       0.73      0.73      0.72       124
weighted avg       0.73      0.73      0.73       124

[Confusion Matrix]
TN	FP
FN	TP
[[49 21]
 [13 41]] 

cutoff = ('Youden', 0.486970763318792)
recall = 0.7592592592592593
precision = 0.6612903225806451
sensitivity = 0.7592592592592593
specificity = 0.7
accuracy = 0.7258064516129032
F1 = 0.7068965517241378
brier = 0.2160423104791631
AUC = 0.7293650793650793
95%CI-AUC = ('Binomial', 0.6422373802567326, 0.8052110565297395)

-----Finish!-----
-----Start Model-----
-----0-----
------test------
              precision    recall  f1-score   support

          NC       0.85      0.79      0.81        14
          RB       0.75      0.82      0.78        11

   micro avg       0.80      0.80      0.80        25
   macro avg       0.80      0.80      0.80        25
weighted avg       0.80      0.80      0.80        25

[Confusion Matrix]
TN	FP
FN	TP
[[11  3]
 [ 2  9]] 

cutoff = ('Youden', 0.5)
recall = 0.8181818181818182
precision = 0.75
sensitivity = 0.8181818181818182
specificity = 0.7857142857142857
accuracy = 0.8
F1 = 0.7826086956521738
brier = 0.18852800000000003
AUC = 0.7987012987012987
95%CI-AUC = ('Binomial', 0.591497055915008, 0.9309022688169337)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

          NC       1.00      1.00      1.00        56
          RB       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.7)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.029481818181818185
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

-----1-----
------test------
              precision    recall  f1-score   support

          NC       0.90      0.64      0.75        14
          RB       0.67      0.91      0.77        11

   micro avg       0.76      0.76      0.76        25
   macro avg       0.78      0.78      0.76        25
weighted avg       0.80      0.76      0.76        25

[Confusion Matrix]
TN	FP
FN	TP
[[ 9  5]
 [ 1 10]] 

cutoff = ('Youden', 0.41)
recall = 0.9090909090909091
precision = 0.6666666666666666
sensitivity = 0.9090909090909091
specificity = 0.6428571428571429
accuracy = 0.76
F1 = 0.7692307692307692
brier = 0.22551200000000002
AUC = 0.7142857142857143
95%CI-AUC = ('Binomial', 0.5001623351600223, 0.8752665811796557)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

          NC       1.00      1.00      1.00        56
          RB       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.64)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.023317171717171715
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

-----2-----
------test------
              precision    recall  f1-score   support

          NC       0.90      0.64      0.75        14
          RB       0.67      0.91      0.77        11

   micro avg       0.76      0.76      0.76        25
   macro avg       0.78      0.78      0.76        25
weighted avg       0.80      0.76      0.76        25

[Confusion Matrix]
TN	FP
FN	TP
[[ 9  5]
 [ 1 10]] 

cutoff = ('Youden', 0.37)
recall = 0.9090909090909091
precision = 0.6666666666666666
sensitivity = 0.9090909090909091
specificity = 0.6428571428571429
accuracy = 0.76
F1 = 0.7692307692307692
brier = 0.24103600000000003
AUC = 0.6915584415584415
95%CI-AUC = ('Binomial', 0.47674078382928714, 0.8589755416055123)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

          NC       1.00      1.00      1.00        56
          RB       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.66)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.02397070707070707
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

-----3-----
------test------
              precision    recall  f1-score   support

          NC       0.81      0.93      0.87        14
          RB       0.89      0.73      0.80        11

   micro avg       0.84      0.84      0.84        25
   macro avg       0.85      0.83      0.83        25
weighted avg       0.85      0.84      0.84        25

[Confusion Matrix]
TN	FP
FN	TP
[[13  1]
 [ 3  8]] 

cutoff = ('Youden', 0.5)
recall = 0.7272727272727273
precision = 0.8888888888888888
sensitivity = 0.7272727272727273
specificity = 0.9285714285714286
accuracy = 0.84
F1 = 0.7999999999999999
brier = 0.15328799999999998
AUC = 0.8474025974025975
95%CI-AUC = ('Binomial', 0.6479714584992858, 0.9585609594568026)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

          NC       1.00      1.00      1.00        56
          RB       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.65)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.028382828282828287
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

-----4-----
------test------
              precision    recall  f1-score   support

          NC       0.90      0.64      0.75        14
          RB       0.64      0.90      0.75        10

   micro avg       0.75      0.75      0.75        24
   macro avg       0.77      0.77      0.75        24
weighted avg       0.79      0.75      0.75        24

[Confusion Matrix]
TN	FP
FN	TP
[[9 5]
 [1 9]] 

cutoff = ('Youden', 0.25)
recall = 0.9
precision = 0.6428571428571429
sensitivity = 0.9
specificity = 0.6428571428571429
accuracy = 0.75
F1 = 0.75
brier = 0.188325
AUC = 0.7642857142857142
95%CI-AUC = ('Binomial', 0.5483099022317911, 0.9115694038401689)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

          NC       1.00      1.00      1.00        56
          RB       1.00      1.00      1.00        44

   micro avg       1.00      1.00      1.00       100
   macro avg       1.00      1.00      1.00       100
weighted avg       1.00      1.00      1.00       100

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 44]] 

cutoff = ('Youden', 0.63)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.025304000000000004
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9637833073548235, nan)

                    0         1         2         3         4      Mean  \
recall       1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
precision    1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
sensitivity  1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
specificity  1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
accuracy     1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
F1           1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
brier        0.029482  0.023317  0.023971  0.028383  0.025304  0.026091   
AUC          1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   

                  Std  
recall       0.000000  
precision    0.000000  
sensitivity  0.000000  
specificity  0.000000  
accuracy     0.000000  
F1           0.000000  
brier        0.002718  
AUC          0.000000  
                    0         1         2         3         4      Mean  \
recall       0.818182  0.909091  0.909091  0.727273  0.900000  0.852727   
precision    0.750000  0.666667  0.666667  0.888889  0.642857  0.723016   
sensitivity  0.818182  0.909091  0.909091  0.727273  0.900000  0.852727   
specificity  0.785714  0.642857  0.642857  0.928571  0.642857  0.728571   
accuracy     0.800000  0.760000  0.760000  0.840000  0.750000  0.782000   
F1           0.782609  0.769231  0.769231  0.800000  0.750000  0.774214   
brier        0.188528  0.225512  0.241036  0.153288  0.188325  0.199338   
AUC          0.798701  0.714286  0.691558  0.847403  0.764286  0.763247   

                  Std  
recall       0.079876  
precision    0.101264  
sensitivity  0.079876  
specificity  0.127775  
accuracy     0.037683  
F1           0.018516  
brier        0.034578  
AUC          0.063001  
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------all------
              precision    recall  f1-score   support

          NC       0.79      0.79      0.79        70
          RB       0.72      0.72      0.72        54

   micro avg       0.76      0.76      0.76       124
   macro avg       0.75      0.75      0.75       124
weighted avg       0.76      0.76      0.76       124

[Confusion Matrix]
TN	FP
FN	TP
[[55 15]
 [15 39]] 

cutoff = ('Youden', 0.5)
recall = 0.7222222222222222
precision = 0.7222222222222222
sensitivity = 0.7222222222222222
specificity = 0.7857142857142857
accuracy = 0.7580645161290323
F1 = 0.7222222222222222
brier = 0.1994266129032258
AUC = 0.7603174603174604
95%CI-AUC = ('Binomial', 0.675409312208698, 0.8323906547827852)

-----Finish!-----
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
In [31]:
# set the structure of inputs for further feature selection
fpath_out = 'TC/Forward_CV-0.85'
if not os.path.exists(fpath_out):os.makedirs(fpath_out)
TC_output_Lasso.typeof = Further_Selection.Forward_CV
TC_output_Lasso.model = lr
TC_output_Lasso.name = name
TC_output_Lasso.fpath = fpath_out

# the further feature selection
TC_output_Further = Further_Selection_CV.selection(TC_output_Lasso,n_splits=5,threshold=0.85)
plt.close('all')

# model developing
for clf,model_name in [(lr,'LR'),(svc,'SVM'),(rf,'RF')]:
    TC_output_Further.model = clf
    TC_output_Further.name = model_name
    TC_output_Further.fpath = fpath_out
    TC_results_CV,TC_result_all = ModelPred.OneCV(TC_output_Further,label=['NC','RB'])
    plt.close()
-----Start Further_Selection_CV-----
-----0-----
-----1-----
-----2-----
-----3-----
-----4-----
-----Finish!-----
-----Start Model-----
-----0-----
------test------
              precision    recall  f1-score   support

          NC       0.75      0.86      0.80        14
          RB       0.78      0.64      0.70        11

   micro avg       0.76      0.76      0.76        25
   macro avg       0.76      0.75      0.75        25
weighted avg       0.76      0.76      0.76        25

[Confusion Matrix]
TN	FP
FN	TP
[[12  2]
 [ 4  7]] 

cutoff = ('Youden', 0.5935665054636224)
recall = 0.6363636363636364
precision = 0.7777777777777778
sensitivity = 0.6363636363636364
specificity = 0.8571428571428571
accuracy = 0.76
F1 = 0.7000000000000001
brier = 0.23967355806862375
AUC = 0.6688311688311688
95%CI-AUC = ('Binomial', 0.4537589122433224, 0.8422050155666465)

------train------
              precision    recall  f1-score   support

          NC       0.90      0.82      0.86        56
          RB       0.79      0.88      0.84        43

   micro avg       0.85      0.85      0.85        99
   macro avg       0.85      0.85      0.85        99
weighted avg       0.85      0.85      0.85        99

[Confusion Matrix]
TN	FP
FN	TP
[[46 10]
 [ 5 38]] 

cutoff = ('Youden', 0.4570045746265311)
recall = 0.8837209302325582
precision = 0.7916666666666666
sensitivity = 0.8837209302325582
specificity = 0.8214285714285714
accuracy = 0.8484848484848485
F1 = 0.8351648351648352
brier = 0.12112781057578377
AUC = 0.9036544850498339
95%CI-AUC = ('Binomial', 0.8277595823694529, 0.9537895961783565)

-----1-----
------test------
              precision    recall  f1-score   support

          NC       1.00      0.50      0.67        14
          RB       0.61      1.00      0.76        11

   micro avg       0.72      0.72      0.72        25
   macro avg       0.81      0.75      0.71        25
weighted avg       0.83      0.72      0.71        25

[Confusion Matrix]
TN	FP
FN	TP
[[ 7  7]
 [ 0 11]] 

cutoff = ('Youden', 0.18430323714442437)
recall = 1.0
precision = 0.6111111111111112
sensitivity = 1.0
specificity = 0.5
accuracy = 0.72
F1 = 0.7586206896551725
brier = 0.23933088816911852
AUC = 0.7337662337662338
95%CI-AUC = ('Binomial', 0.5206066641883598, 0.8888229898052754)

------train------
              precision    recall  f1-score   support

          NC       0.87      0.86      0.86        56
          RB       0.82      0.84      0.83        43

   micro avg       0.85      0.85      0.85        99
   macro avg       0.85      0.85      0.85        99
weighted avg       0.85      0.85      0.85        99

[Confusion Matrix]
TN	FP
FN	TP
[[48  8]
 [ 7 36]] 

cutoff = ('Youden', 0.4692026255873121)
recall = 0.8372093023255814
precision = 0.8181818181818182
sensitivity = 0.8372093023255814
specificity = 0.8571428571428571
accuracy = 0.8484848484848485
F1 = 0.8275862068965518
brier = 0.1356711309972889
AUC = 0.8799833887043189
95%CI-AUC = ('Binomial', 0.7992560281640527, 0.9366604941047192)

-----2-----
------test------
              precision    recall  f1-score   support

          NC       1.00      0.57      0.73        14
          RB       0.65      1.00      0.79        11

   micro avg       0.76      0.76      0.76        25
   macro avg       0.82      0.79      0.76        25
weighted avg       0.84      0.76      0.75        25

[Confusion Matrix]
TN	FP
FN	TP
[[ 8  6]
 [ 0 11]] 

cutoff = ('Youden', 0.3325332929635006)
recall = 1.0
precision = 0.6470588235294118
sensitivity = 1.0
specificity = 0.5714285714285714
accuracy = 0.76
F1 = 0.7857142857142858
brier = 0.1909833301648981
AUC = 0.8051948051948052
95%CI-AUC = ('Binomial', 0.5988452401884774, 0.9348091837360017)

------train------
              precision    recall  f1-score   support

          NC       0.84      0.86      0.85        56
          RB       0.81      0.79      0.80        43

   micro avg       0.83      0.83      0.83        99
   macro avg       0.83      0.82      0.82        99
weighted avg       0.83      0.83      0.83        99

[Confusion Matrix]
TN	FP
FN	TP
[[48  8]
 [ 9 34]] 

cutoff = ('Youden', 0.45693721422475736)
recall = 0.7906976744186046
precision = 0.8095238095238095
sensitivity = 0.7906976744186046
specificity = 0.8571428571428571
accuracy = 0.8282828282828283
F1 = 0.8
brier = 0.14436349129027587
AUC = 0.8754152823920266
95%CI-AUC = ('Binomial', 0.793845177925711, 0.9332555462283807)

-----3-----
------test------
              precision    recall  f1-score   support

          NC       0.86      0.86      0.86        14
          RB       0.82      0.82      0.82        11

   micro avg       0.84      0.84      0.84        25
   macro avg       0.84      0.84      0.84        25
weighted avg       0.84      0.84      0.84        25

[Confusion Matrix]
TN	FP
FN	TP
[[12  2]
 [ 2  9]] 

cutoff = ('Youden', 0.599799407845647)
recall = 0.8181818181818182
precision = 0.8181818181818182
sensitivity = 0.8181818181818182
specificity = 0.8571428571428571
accuracy = 0.84
F1 = 0.8181818181818182
brier = 0.14674895879763852
AUC = 0.8441558441558441
95%CI-AUC = ('Binomial', 0.6441014717388924, 0.9568456190770979)

------train------
              precision    recall  f1-score   support

          NC       0.81      0.86      0.83        56
          RB       0.80      0.74      0.77        43

   micro avg       0.81      0.81      0.81        99
   macro avg       0.81      0.80      0.80        99
weighted avg       0.81      0.81      0.81        99

[Confusion Matrix]
TN	FP
FN	TP
[[48  8]
 [11 32]] 

cutoff = ('Youden', 0.4828510738214207)
recall = 0.7441860465116279
precision = 0.8
sensitivity = 0.7441860465116279
specificity = 0.8571428571428571
accuracy = 0.8080808080808081
F1 = 0.7710843373493975
brier = 0.14429198120010017
AUC = 0.8745847176079734
95%CI-AUC = ('Binomial', 0.7928642047132081, 0.9326333773965327)

-----4-----
------test------
              precision    recall  f1-score   support

          NC       0.81      0.93      0.87        14
          RB       0.88      0.70      0.78        10

   micro avg       0.83      0.83      0.83        24
   macro avg       0.84      0.81      0.82        24
weighted avg       0.84      0.83      0.83        24

[Confusion Matrix]
TN	FP
FN	TP
[[13  1]
 [ 3  7]] 

cutoff = ('Youden', 0.4896735575704285)
recall = 0.7
precision = 0.875
sensitivity = 0.7
specificity = 0.9285714285714286
accuracy = 0.8333333333333334
F1 = 0.7777777777777777
brier = 0.17676127157189805
AUC = 0.8
95%CI-AUC = ('Binomial', 0.5878433872065862, 0.9336879520344967)

------train------
              precision    recall  f1-score   support

          NC       0.87      0.82      0.84        56
          RB       0.79      0.84      0.81        44

   micro avg       0.83      0.83      0.83       100
   macro avg       0.83      0.83      0.83       100
weighted avg       0.83      0.83      0.83       100

[Confusion Matrix]
TN	FP
FN	TP
[[46 10]
 [ 7 37]] 

cutoff = ('Youden', 0.4150308181605387)
recall = 0.8409090909090909
precision = 0.7872340425531915
sensitivity = 0.8409090909090909
specificity = 0.8214285714285714
accuracy = 0.83
F1 = 0.8131868131868133
brier = 0.14801197153699552
AUC = 0.8741883116883117
95%CI-AUC = ('Binomial', 0.7928886657778024, 0.9320862729002128)

                    0         1         2         3         4      Mean  \
recall       0.883721  0.837209  0.790698  0.744186  0.840909  0.819345   
precision    0.791667  0.818182  0.809524  0.800000  0.787234  0.801321   
sensitivity  0.883721  0.837209  0.790698  0.744186  0.840909  0.819345   
specificity  0.821429  0.857143  0.857143  0.857143  0.821429  0.842857   
accuracy     0.848485  0.848485  0.828283  0.808081  0.830000  0.832667   
F1           0.835165  0.827586  0.800000  0.771084  0.813187  0.809404   
brier        0.121128  0.135671  0.144363  0.144292  0.148012  0.138693   
AUC          0.903654  0.879983  0.875415  0.874585  0.874188  0.881565   

                  Std  
recall       0.053381  
precision    0.012698  
sensitivity  0.053381  
specificity  0.019562  
accuracy     0.016817  
F1           0.025325  
brier        0.010817  
AUC          0.012564  
                    0         1         2         3         4      Mean  \
recall       0.636364  1.000000  1.000000  0.818182  0.700000  0.830909   
precision    0.777778  0.611111  0.647059  0.818182  0.875000  0.745826   
sensitivity  0.636364  1.000000  1.000000  0.818182  0.700000  0.830909   
specificity  0.857143  0.500000  0.571429  0.857143  0.928571  0.742857   
accuracy     0.760000  0.720000  0.760000  0.840000  0.833333  0.782667   
F1           0.700000  0.758621  0.785714  0.818182  0.777778  0.768059   
brier        0.239674  0.239331  0.190983  0.146749  0.176761  0.198700   
AUC          0.668831  0.733766  0.805195  0.844156  0.800000  0.770390   

                  Std  
recall       0.167579  
precision    0.112744  
sensitivity  0.167579  
specificity  0.192989  
accuracy     0.051983  
F1           0.043702  
brier        0.040526  
AUC          0.069255  
------all------
              precision    recall  f1-score   support

          NC       0.78      0.77      0.78        70
          RB       0.71      0.72      0.72        54

   micro avg       0.75      0.75      0.75       124
   macro avg       0.75      0.75      0.75       124
weighted avg       0.75      0.75      0.75       124

[Confusion Matrix]
TN	FP
FN	TP
[[54 16]
 [15 39]] 

cutoff = ('Youden', 0.4896735575704285)
recall = 0.7222222222222222
precision = 0.7090909090909091
sensitivity = 0.7222222222222222
specificity = 0.7714285714285715
accuracy = 0.75
F1 = 0.7155963302752293
brier = 0.19887652336881073
AUC = 0.7687830687830687
95%CI-AUC = ('Binomial', 0.6845699159150108, 0.8397337364041085)

-----Finish!-----
-----Start Model-----
-----0-----
------test------
              precision    recall  f1-score   support

          NC       0.78      1.00      0.88        14
          RB       1.00      0.64      0.78        11

   micro avg       0.84      0.84      0.84        25
   macro avg       0.89      0.82      0.83        25
weighted avg       0.88      0.84      0.83        25

[Confusion Matrix]
TN	FP
FN	TP
[[14  0]
 [ 4  7]] 

cutoff = ('Youden', 0.5960953444917525)
recall = 0.6363636363636364
precision = 1.0
sensitivity = 0.6363636363636364
specificity = 1.0
accuracy = 0.84
F1 = 0.7777777777777778
brier = 0.17933908693714762
AUC = 0.7792207792207791
95%CI-AUC = ('Binomial', 0.5697525091143241, 0.9188286472222847)

------train------
              precision    recall  f1-score   support

          NC       0.91      0.93      0.92        56
          RB       0.90      0.88      0.89        43

   micro avg       0.91      0.91      0.91        99
   macro avg       0.91      0.91      0.91        99
weighted avg       0.91      0.91      0.91        99

[Confusion Matrix]
TN	FP
FN	TP
[[52  4]
 [ 5 38]] 

cutoff = ('Youden', 0.5145698259956726)
recall = 0.8837209302325582
precision = 0.9047619047619048
sensitivity = 0.8837209302325582
specificity = 0.9285714285714286
accuracy = 0.9090909090909091
F1 = 0.8941176470588236
brier = 0.09517465706819654
AUC = 0.9472591362126246
95%CI-AUC = ('Binomial', 0.8830855404270885, 0.9821168222995504)

-----1-----
------test------
              precision    recall  f1-score   support

          NC       0.89      0.57      0.70        14
          RB       0.62      0.91      0.74        11

   micro avg       0.72      0.72      0.72        25
   macro avg       0.76      0.74      0.72        25
weighted avg       0.77      0.72      0.72        25

[Confusion Matrix]
TN	FP
FN	TP
[[ 8  6]
 [ 1 10]] 

cutoff = ('Youden', 0.28318586234196486)
recall = 0.9090909090909091
precision = 0.625
sensitivity = 0.9090909090909091
specificity = 0.5714285714285714
accuracy = 0.72
F1 = 0.7407407407407406
brier = 0.2184718045516427
AUC = 0.7207792207792207
95%CI-AUC = ('Binomial', 0.5069383185338913, 0.8798285008906752)

------train------
              precision    recall  f1-score   support

          NC       0.91      0.88      0.89        56
          RB       0.84      0.88      0.86        43

   micro avg       0.88      0.88      0.88        99
   macro avg       0.88      0.88      0.88        99
weighted avg       0.88      0.88      0.88        99

[Confusion Matrix]
TN	FP
FN	TP
[[49  7]
 [ 5 38]] 

cutoff = ('Youden', 0.4387797812177666)
recall = 0.8837209302325582
precision = 0.8444444444444444
sensitivity = 0.8837209302325582
specificity = 0.875
accuracy = 0.8787878787878788
F1 = 0.8636363636363636
brier = 0.1040112765268145
AUC = 0.9343853820598007
95%CI-AUC = ('Binomial', 0.8662562030644628, 0.9743361484128362)

-----2-----
------test------
              precision    recall  f1-score   support

          NC       0.91      0.71      0.80        14
          RB       0.71      0.91      0.80        11

   micro avg       0.80      0.80      0.80        25
   macro avg       0.81      0.81      0.80        25
weighted avg       0.82      0.80      0.80        25

[Confusion Matrix]
TN	FP
FN	TP
[[10  4]
 [ 1 10]] 

cutoff = ('Youden', 0.4305939014452084)
recall = 0.9090909090909091
precision = 0.7142857142857143
sensitivity = 0.9090909090909091
specificity = 0.7142857142857143
accuracy = 0.8
F1 = 0.8
brier = 0.19120369295528428
AUC = 0.8311688311688312
95%CI-AUC = ('Binomial', 0.6287807960115356, 0.9497868286347726)

------train------
              precision    recall  f1-score   support

          NC       0.87      0.84      0.85        56
          RB       0.80      0.84      0.82        43

   micro avg       0.84      0.84      0.84        99
   macro avg       0.84      0.84      0.84        99
weighted avg       0.84      0.84      0.84        99

[Confusion Matrix]
TN	FP
FN	TP
[[47  9]
 [ 7 36]] 

cutoff = ('Youden', 0.36881709710346)
recall = 0.8372093023255814
precision = 0.8
sensitivity = 0.8372093023255814
specificity = 0.8392857142857143
accuracy = 0.8383838383838383
F1 = 0.8181818181818183
brier = 0.13467755396317513
AUC = 0.8874584717607974
95%CI-AUC = ('Binomial', 0.8081692052211917, 0.9421672801219797)

-----3-----
------test------
              precision    recall  f1-score   support

          NC       0.92      0.86      0.89        14
          RB       0.83      0.91      0.87        11

   micro avg       0.88      0.88      0.88        25
   macro avg       0.88      0.88      0.88        25
weighted avg       0.88      0.88      0.88        25

[Confusion Matrix]
TN	FP
FN	TP
[[12  2]
 [ 1 10]] 

cutoff = ('Youden', 0.4941039838830814)
recall = 0.9090909090909091
precision = 0.8333333333333334
sensitivity = 0.9090909090909091
specificity = 0.8571428571428571
accuracy = 0.88
F1 = 0.8695652173913043
brier = 0.13226585537413196
AUC = 0.9025974025974025
95%CI-AUC = ('Binomial', 0.716645614645395, 0.9840107496477167)

------train------
              precision    recall  f1-score   support

          NC       0.88      0.91      0.89        56
          RB       0.88      0.84      0.86        43

   micro avg       0.88      0.88      0.88        99
   macro avg       0.88      0.87      0.88        99
weighted avg       0.88      0.88      0.88        99

[Confusion Matrix]
TN	FP
FN	TP
[[51  5]
 [ 7 36]] 

cutoff = ('Youden', 0.4712907220618809)
recall = 0.8372093023255814
precision = 0.8780487804878049
sensitivity = 0.8372093023255814
specificity = 0.9107142857142857
accuracy = 0.8787878787878788
F1 = 0.8571428571428572
brier = 0.11566416773229307
AUC = 0.90656146179402
95%CI-AUC = ('Binomial', 0.8313204847049749, 0.9558256014013052)

-----4-----
------test------
              precision    recall  f1-score   support

          NC       0.81      0.93      0.87        14
          RB       0.88      0.70      0.78        10

   micro avg       0.83      0.83      0.83        24
   macro avg       0.84      0.81      0.82        24
weighted avg       0.84      0.83      0.83        24

[Confusion Matrix]
TN	FP
FN	TP
[[13  1]
 [ 3  7]] 

cutoff = ('Youden', 0.6188785849320634)
recall = 0.7
precision = 0.875
sensitivity = 0.7
specificity = 0.9285714285714286
accuracy = 0.8333333333333334
F1 = 0.7777777777777777
brier = 0.16030590494717958
AUC = 0.8857142857142857
95%CI-AUC = ('Binomial', 0.6898089360463363, 0.9781363807975156)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

          NC       0.92      0.82      0.87        56
          RB       0.80      0.91      0.85        44

   micro avg       0.86      0.86      0.86       100
   macro avg       0.86      0.87      0.86       100
weighted avg       0.87      0.86      0.86       100

[Confusion Matrix]
TN	FP
FN	TP
[[46 10]
 [ 4 40]] 

cutoff = ('Youden', 0.4515961376097948)
recall = 0.9090909090909091
precision = 0.8
sensitivity = 0.9090909090909091
specificity = 0.8214285714285714
accuracy = 0.86
F1 = 0.8510638297872342
brier = 0.11153252143106149
AUC = 0.9310064935064934
95%CI-AUC = ('Binomial', 0.8623654733618611, 0.9720401978261864)

                    0         1         2         3         4      Mean  \
recall       0.883721  0.883721  0.837209  0.837209  0.909091  0.870190   
precision    0.904762  0.844444  0.800000  0.878049  0.800000  0.845451   
sensitivity  0.883721  0.883721  0.837209  0.837209  0.909091  0.870190   
specificity  0.928571  0.875000  0.839286  0.910714  0.821429  0.875000   
accuracy     0.909091  0.878788  0.838384  0.878788  0.860000  0.873010   
F1           0.894118  0.863636  0.818182  0.857143  0.851064  0.856829   
brier        0.095175  0.104011  0.134678  0.115664  0.111533  0.112212   
AUC          0.947259  0.934385  0.887458  0.906561  0.931006  0.921334   

                  Std  
recall       0.031839  
precision    0.046672  
sensitivity  0.031839  
specificity  0.045527  
accuracy     0.026157  
F1           0.027220  
brier        0.014787  
AUC          0.023988  
                    0         1         2         3         4      Mean  \
recall       0.636364  0.909091  0.909091  0.909091  0.700000  0.812727   
precision    1.000000  0.625000  0.714286  0.833333  0.875000  0.809524   
sensitivity  0.636364  0.909091  0.909091  0.909091  0.700000  0.812727   
specificity  1.000000  0.571429  0.714286  0.857143  0.928571  0.814286   
accuracy     0.840000  0.720000  0.800000  0.880000  0.833333  0.814667   
F1           0.777778  0.740741  0.800000  0.869565  0.777778  0.793172   
brier        0.179339  0.218472  0.191204  0.132266  0.160306  0.176317   
AUC          0.779221  0.720779  0.831169  0.902597  0.885714  0.823896   

                  Std  
recall       0.133856  
precision    0.145133  
sensitivity  0.133856  
specificity  0.172023  
accuracy     0.060074  
F1           0.047711  
brier        0.032422  
AUC          0.075328  
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------all------
              precision    recall  f1-score   support

          NC       0.76      0.86      0.81        70
          RB       0.78      0.65      0.71        54

   micro avg       0.77      0.77      0.77       124
   macro avg       0.77      0.75      0.76       124
weighted avg       0.77      0.77      0.76       124

[Confusion Matrix]
TN	FP
FN	TP
[[60 10]
 [19 35]] 

cutoff = ('Youden', 0.5960953444917525)
recall = 0.6481481481481481
precision = 0.7777777777777778
sensitivity = 0.6481481481481481
specificity = 0.8571428571428571
accuracy = 0.7661290322580645
F1 = 0.707070707070707
brier = 0.1764463928563506
AUC = 0.8103174603174603
95%CI-AUC = ('Binomial', 0.730129771117034, 0.8751222495948888)

-----Finish!-----
-----Start Model-----
-----0-----
------test------
              precision    recall  f1-score   support

          NC       0.86      0.86      0.86        14
          RB       0.82      0.82      0.82        11

   micro avg       0.84      0.84      0.84        25
   macro avg       0.84      0.84      0.84        25
weighted avg       0.84      0.84      0.84        25

[Confusion Matrix]
TN	FP
FN	TP
[[12  2]
 [ 2  9]] 

cutoff = ('Youden', 0.56)
recall = 0.8181818181818182
precision = 0.8181818181818182
sensitivity = 0.8181818181818182
specificity = 0.8571428571428571
accuracy = 0.84
F1 = 0.8181818181818182
brier = 0.17494800000000002
AUC = 0.8116883116883118
95%CI-AUC = ('Binomial', 0.6062459490380402, 0.9386536573797263)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

          NC       1.00      1.00      1.00        56
          RB       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.64)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.0269020202020202
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

-----1-----
------test------
              precision    recall  f1-score   support

          NC       0.79      0.79      0.79        14
          RB       0.73      0.73      0.73        11

   micro avg       0.76      0.76      0.76        25
   macro avg       0.76      0.76      0.76        25
weighted avg       0.76      0.76      0.76        25

[Confusion Matrix]
TN	FP
FN	TP
[[11  3]
 [ 3  8]] 

cutoff = ('Youden', 0.57)
recall = 0.7272727272727273
precision = 0.7272727272727273
sensitivity = 0.7272727272727273
specificity = 0.7857142857142857
accuracy = 0.76
F1 = 0.7272727272727273
brier = 0.195104
AUC = 0.7792207792207793
95%CI-AUC = ('Binomial', 0.569752509114324, 0.9188286472222847)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------train------
              precision    recall  f1-score   support

          NC       1.00      1.00      1.00        56
          RB       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.59)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.022625252525252524
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

-----2-----
------test------
              precision    recall  f1-score   support

          NC       0.89      0.57      0.70        14
          RB       0.62      0.91      0.74        11

   micro avg       0.72      0.72      0.72        25
   macro avg       0.76      0.74      0.72        25
weighted avg       0.77      0.72      0.72        25

[Confusion Matrix]
TN	FP
FN	TP
[[ 8  6]
 [ 1 10]] 

cutoff = ('Youden', 0.4)
recall = 0.9090909090909091
precision = 0.625
sensitivity = 0.9090909090909091
specificity = 0.5714285714285714
accuracy = 0.72
F1 = 0.7407407407407406
brier = 0.183032
AUC = 0.8084415584415585
95%CI-AUC = ('Binomial', 0.6025389264488501, 0.9367393742358567)

------train------
              precision    recall  f1-score   support

          NC       1.00      1.00      1.00        56
          RB       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.58)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.026202020202020206
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
-----3-----
------test------
              precision    recall  f1-score   support

          NC       0.82      1.00      0.90        14
          RB       1.00      0.73      0.84        11

   micro avg       0.88      0.88      0.88        25
   macro avg       0.91      0.86      0.87        25
weighted avg       0.90      0.88      0.88        25

[Confusion Matrix]
TN	FP
FN	TP
[[14  0]
 [ 3  8]] 

cutoff = ('Youden', 0.5)
recall = 0.7272727272727273
precision = 1.0
sensitivity = 0.7272727272727273
specificity = 1.0
accuracy = 0.88
F1 = 0.8421052631578948
brier = 0.157724
AUC = 0.827922077922078
95%CI-AUC = ('Binomial', 0.6249889329318279, 0.9479750440455293)

------train------
              precision    recall  f1-score   support

          NC       1.00      1.00      1.00        56
          RB       1.00      1.00      1.00        43

   micro avg       1.00      1.00      1.00        99
   macro avg       1.00      1.00      1.00        99
weighted avg       1.00      1.00      1.00        99

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 43]] 

cutoff = ('Youden', 0.6)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.026823232323232322
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9634242550165211, nan)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
-----4-----
------test------
              precision    recall  f1-score   support

          NC       0.91      0.71      0.80        14
          RB       0.69      0.90      0.78        10

   micro avg       0.79      0.79      0.79        24
   macro avg       0.80      0.81      0.79        24
weighted avg       0.82      0.79      0.79        24

[Confusion Matrix]
TN	FP
FN	TP
[[10  4]
 [ 1  9]] 

cutoff = ('Youden', 0.06)
recall = 0.9
precision = 0.6923076923076923
sensitivity = 0.9
specificity = 0.7142857142857143
accuracy = 0.7916666666666666
F1 = 0.7826086956521738
brier = 0.17835416666666667
AUC = 0.8535714285714286
95%CI-AUC = ('Binomial', 0.6501954948532898, 0.9632044978249255)

------train------
              precision    recall  f1-score   support

          NC       1.00      1.00      1.00        56
          RB       1.00      1.00      1.00        44

   micro avg       1.00      1.00      1.00       100
   macro avg       1.00      1.00      1.00       100
weighted avg       1.00      1.00      1.00       100

[Confusion Matrix]
TN	FP
FN	TP
[[56  0]
 [ 0 44]] 

cutoff = ('Youden', 0.63)
recall = 1.0
precision = 1.0
sensitivity = 1.0
specificity = 1.0
accuracy = 1.0
F1 = 1.0
brier = 0.02742300000000001
AUC = 1.0
95%CI-AUC = ('Binomial', 0.9637833073548235, nan)

/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
                    0         1         2         3         4      Mean  \
recall       1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
precision    1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
sensitivity  1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
specificity  1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
accuracy     1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
F1           1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   
brier        0.026902  0.022625  0.026202  0.026823  0.027423  0.025995   
AUC          1.000000  1.000000  1.000000  1.000000  1.000000  1.000000   

                  Std  
recall       0.000000  
precision    0.000000  
sensitivity  0.000000  
specificity  0.000000  
accuracy     0.000000  
F1           0.000000  
brier        0.001933  
AUC          0.000000  
                    0         1         2         3         4      Mean  \
recall       0.818182  0.727273  0.909091  0.727273  0.900000  0.816364   
precision    0.818182  0.727273  0.625000  1.000000  0.692308  0.772552   
sensitivity  0.818182  0.727273  0.909091  0.727273  0.900000  0.816364   
specificity  0.857143  0.785714  0.571429  1.000000  0.714286  0.785714   
accuracy     0.840000  0.760000  0.720000  0.880000  0.791667  0.798333   
F1           0.818182  0.727273  0.740741  0.842105  0.782609  0.782182   
brier        0.174948  0.195104  0.183032  0.157724  0.178354  0.177832   
AUC          0.811688  0.779221  0.808442  0.827922  0.853571  0.816169   

                  Std  
recall       0.088700  
precision    0.144979  
sensitivity  0.088700  
specificity  0.159719  
accuracy     0.063355  
F1           0.049039  
brier        0.013586  
AUC          0.027304  
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
/home/huiying/anaconda3/lib/python3.7/site-packages/matplotlib/pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)
------all------
              precision    recall  f1-score   support

          NC       0.78      0.87      0.82        70
          RB       0.80      0.69      0.74        54

   micro avg       0.79      0.79      0.79       124
   macro avg       0.79      0.78      0.78       124
weighted avg       0.79      0.79      0.79       124

[Confusion Matrix]
TN	FP
FN	TP
[[61  9]
 [17 37]] 

cutoff = ('Youden', 0.56)
recall = 0.6851851851851852
precision = 0.8043478260869565
sensitivity = 0.6851851851851852
specificity = 0.8714285714285714
accuracy = 0.7903225806451613
F1 = 0.74
brier = 0.1778282258064516
AUC = 0.8072751322751323
95%CI-AUC = ('Binomial', 0.7267550905482664, 0.8725691357364078)

-----Finish!-----
In [ ]: